gemverse 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 69db10da33808ff62ca728d7ada452de1283a465d9fbbce8b9919d88f976a9b4
+  data.tar.gz: 1554b1c90f9c0e467bee55f2e98eff8ffae13e3b2090d8bd5bb21ab86b81b645
+SHA512:
+  metadata.gz: 4d640f9f26100ae1646431670fa275bf267bb9ab88727d754e0c3efffaa29c74c2ea1136da0516908de48020aead49e38aed93780a917c85c04361e84ac3b518
+  data.tar.gz: 5fbf83a842f797c19b8054aa1b86c3891650bf5a1a6e9bb85dda0371755713fe38d0321db4c653a70951f72e64c5017eea6e831158d4ac1b6c2fdfdf4346ef38

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+### 0.0.1 / 2023-01-20
+
+* Everything is new. First release.

data/Manifest.txt

@@ -0,0 +1,9 @@
+CHANGELOG.md
+Manifest.txt
+README.md
+Rakefile
+lib/gemverse.rb
+lib/gemverse/api.rb
+lib/gemverse/cache.rb
+lib/gemverse/gems.rb
+lib/gemverse/timeline.rb

data/README.md

@@ -0,0 +1,123 @@
+# Gemverse - Gem Universe
+
+gemverse gem - gem universe incl. rubygems API V1 wrapper lite; gem version cache, gem timeline reports, 'n' more
+
+
+
+* home  :: [github.com/rubycocos/git](https://github.com/rubycocos/git)
+* bugs  :: [github.com/rubycocos/git/issues](https://github.com/rubycocos/git/issues)
+* gem   :: [rubygems.org/gems/gemverse](https://rubygems.org/gems/gemverse)
+* rdoc  :: [rubydoc.info/gems/gemverse](http://rubydoc.info/gems/gemverse)
+
+
+
+
+
+## Usage
+
+
+### RubyGems API "To The Metal" Wrapper V1 - Lite Edition
+
+The gemverse includes a lightweight "to the metal"
+wrapper for the rubygems API V1
+that returns data(sets) in the JSON format:
+
+``` ruby
+require 'gemverse'
+
+## get all gems owned by the author with the handle / known as gettalong
+data = Gems::API.gems_by( 'gettalong' )
+# same as https://rubygems.org/api/v1/owners/gettalong/gems.json
+
+## get all versions of the hexapdf gem
+data = Gems::API.versions( 'hexapdf' )
+# same as https://rubygems.org/api/v1/versions/hexpdf.json"
+
+#...
+```
+
+
+### Gem Cache 'n' Timeline Reports
+
+Let's build a gem timeline report / what's news page.
+Let's spotlight the work of [Thomas Leitner, Austria also known as `gettalong`](https://rubygems.org/profiles/gettalong)
+who 24 published gems (as of 2023) in the last 10+ years.
+
+Note:  Yes, you can. Replace the `gettalong`  rubygems id / login with your own to build your very own timeline.
+
+
+**Step 1 - Online - Get gems & versions via "higher-level" rubygems api calls**
+
+``` ruby
+cache = Gems::Cache.new( './cache' )
+
+gems = Gems.find_by( owner: 'gettalong' )
+puts "  #{gems.size} record(s)"
+
+## bonus: save gems in a "flat" tabular datafile using the comma-separated values (.csv) format
+gems.export( './profile/gettalong/gems.csv' )
+
+## fetch all gem versions and (auto-save)
+##   in a "flat" tabular datafile (e.g. <gem>/versions.csv)
+##    using the comma-spearated values (.csv) format
+##    in the cache directory
+cache.update_versions( gems: gems )
+```
+
+
+**Step 2 - Offline - Read versions from cache and build reports / timeline**
+
+``` ruby
+cache = Gems::Cache.new( './gems' )
+
+gems = read_csv( './profile/gettalong/gems.csv' )
+puts "  #{gems.size} record(s)"
+
+versions = cache.read_versions( gems: gems )
+puts "  #{versions.size} record(s)"
+
+timeline = Gems::Timeline.new( versions,
+                               title: "Thomas Leitner's Timeline" )
+timeline.save( "./profile/gettalong/README.md" )
+```
+
+
+That's it.
+
+
+
+Tip:  You can build "custom" timeline reports
+and filter / select the gems to include as you like.
+Let's (re)build the timeline for all ruby cocos (code commons)
+gems.
+
+``` ruby
+cache = Gems::Cache.new( './cache' )
+
+gems = read_csv( './collection/cocos.csv' )
+puts "  #{gems.size} record(s)"
+
+versions = cache.read_versions( gems: gems )
+puts "   #{versions.size} record(s)"
+
+timeline = Gems::Timeline.new( versions,
+                               title: 'Ruby Code Commons (COCOS) Timeline' )
+timeline.save( "./collection/cocos/README.md" )
+```
+
+That's it.
+
+
+See
+[Thomas Leitner's Timeline](samples/gems_gettalong),
+[Jan Lelis's Timeline](samples/gems_janlelis),
+[Ruby Code Commons (COCOS) Timeline](samples/gems_cocos), and some more
+for some real-world timeline samples.
+
+
+
+## License
+
+The `gemverse` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+

data/Rakefile

@@ -0,0 +1,28 @@
+require 'hoe'
+
+Hoe.spec 'gemverse' do
+
+  self.version = '0.0.1'   # note: for now add version inline
+
+  self.summary = "gemverse gem - gem universe incl. rubygems API V1 wrapper lite; gem version cache, gem timeline reports, 'n' more"
+  self.description = summary
+
+  self.urls = { home: 'https://github.com/rubycocos/git' }
+
+  self.author = 'Gerald Bauer'
+  self.email = 'opensport@googlegroups.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'CHANGELOG.md'
+
+  self.licenses = ['Public Domain']
+
+  self.extra_deps = [
+    ['cocos'],
+  ]
+
+  self.spec_extras = {
+    required_ruby_version: '>= 2.2.2'
+  }
+end

data/lib/gemverse/api.rb

@@ -0,0 +1,42 @@
+
+module Gems
+module API
+
+BASE = 'https://rubygems.org/api/v1'
+
+def self.gems_by( owner )
+  src = "#{BASE}/owners/#{owner}/gems.json"
+  call( src )
+end
+
+
+def self.versions( name )
+  ## note: will NOT include yanked versions
+  ##   check if there's a query parameter ???
+  src = "#{BASE}/versions/#{name}.json"
+  call( src )
+end
+
+
+def self.call( src )   ## get response as (parsed) json (hash table)
+
+  headers = {
+    # 'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
+    'User-Agent' => "ruby v#{RUBY_VERSION}",
+   }
+
+  response = Webclient.get( src, headers: headers )
+
+  if response.status.ok?
+    puts "#{response.status.code} #{response.status.message} -  content_type: #{response.content_type}, content_length: #{response.content_length}"
+
+    response.json
+  else
+    puts "!! HTTP ERROR:"
+    puts "#{response.status.code} #{response.status.message}"
+    exit 1
+  end
+end
+
+end  # module API
+end  # module Gems

data/lib/gemverse/cache.rb

@@ -0,0 +1,100 @@
+
+
+module Gems
+class Cache
+def initialize( basedir='./gems' )
+  @basedir = basedir
+end
+
+
+def update_versions( gems: [] )
+
+  delay_in_s = 0.5
+
+  gems.each_with_index do |gem,i|
+
+    name = if gem.is_a?( String )
+                gem
+           elsif gem.is_a?( Hash )
+                gem['name']
+           else  ## assume our own Gem struct/class for now
+                gem.name
+           end
+
+    puts "==>  #{i+1}/#{gems.size} #{name}..."
+
+    ## update versions info
+    puts "  sleeping #{delay_in_s} second(s)"
+    sleep( delay_in_s )
+
+    data = Gems::API.versions( name )
+    puts "   #{data.size} version record(s) found"
+
+    headers = ['name',
+               'version',
+               'created',
+               'downloads']
+    recs = []
+    data.each do |h|
+      ## only use year/month/day for now now hours etc.
+      created = Date.strptime( h['created_at'], '%Y-%m-%d' )
+
+       rec = [
+         name,
+         h['number'],
+         created.strftime( '%Y-%m-%d' ),
+         h['downloads_count'].to_s,
+       ]
+       recs << rec
+    end
+    write_csv( "#{@basedir}/#{name}/versions.csv", [headers]+recs )
+  end
+end
+
+
+
+def read_versions( gems: [] )
+  ## read in and merge all version records
+versions = []
+
+ ## if no gems passed in - get all versions.csv datasets in basedir
+ if gems.empty?
+   paths = Dir.glob( "#{@basedir}/*/versions.csv" )
+   gems = paths.map { |path| File.basename(File.dirname(path)) }
+ end
+
+  gems.each_with_index do |gem,i|
+
+    name = if gem.is_a?( String )
+                gem
+           elsif gem.is_a?( Hash )
+                gem['name']
+           else  ## assume our own Gem struct/class for now
+                gem.name
+           end
+
+    path = "#{@basedir}/#{name}/versions.csv"
+    puts "==> #{i+1}/#{gems.size} reading #{name}..."
+    recs = read_csv( path )
+    recs.reverse.each_with_index do |rec,n|
+        more = { 'count' => (n+1).to_s }   ## auto-add version count(er) e.g. 1,2,3,...
+        versions <<  rec.merge( more )
+    end
+    puts "   #{recs.size} record(s)"
+  end
+
+  ## sort by version created and name
+  versions = versions.sort do |l,r|
+    res =  r['created'] <=> l['created']
+    res =  l['name']    <=> r['name']     if res == 0
+    res =  r['version'] <=> l['version']  if res == 0
+    res
+  end
+
+  versions
+end
+
+
+
+end   ## class Cache
+end   ## module Gems

data/lib/gemverse/gems.rb

@@ -0,0 +1,105 @@
+
+module Gems
+
+
+
+class GemDataset    ## rename to Gems or Gemset or such - why? why not?
+
+
+  def initialize( gems )
+     @gems = gems
+  end
+
+  def size() @gems.size; end
+  def each_with_index( &block ) @gems.each_with_index( &block ); end
+  def each( &block )            @gems.each( &block );            end
+
+
+  ## todo/check:  add an write_csv( path ) alias / alternate method name - why? why not?
+def export( path )
+  recs = []
+
+  @gems.each do |gem|
+
+    if gem.yanked?
+      puts "!! ERROR - includes yanked gem"
+      pp gem
+      exit 1
+    end
+
+    rec = [gem.name,
+           gem.version,
+           gem.version_created.strftime( '%Y-%m-%d' ),
+           gem.version_downloads.to_s,
+           gem.homepage,
+           gem.runtime_dependencies.join( ' | ' ),
+          ]
+    recs << rec
+  end
+  headers = ['name',
+             'version',
+             'version_created',
+             'version_downloads',
+             'homepage',
+             'dependencies',
+            ]
+  write_csv( path, [headers]+recs )
+end # method export
+end   ## class GemDataset
+
+
+
+
+class Gem   ## todo/check: rename or use Gem::Meta or such - why? why not?
+
+  attr_reader :name,
+              :version, :version_created, :version_downloads,
+              :homepage,
+              :runtime
+
+
+  def self.create( h )  ## todo/check: rename to  create_from_json or such - why? why not?
+      ## optional keyword args
+      kwargs = {
+        version:           h['version'],
+        ## only use year/month/day for now now hours etc. - why? why not
+        version_created:   h['version_created_at'] ? Date.strptime( h['version_created_at'], '%Y-%m-%d' ) : nil,
+        version_downloads: h['version_downloads'],
+        ## note: (auto-)clean
+        ##   for example - newline seen in "http://icanhasaudio.com/\n" !!!
+        homepage: h['homepage_uri'] ? h['homepage_uri'].gsub( /[ \r\n]/, '')
+        : nil,
+        yanked:   h['yanked'],
+        runtime:  h['dependencies']['runtime'].map { |dep| dep['name'] }
+      }
+
+      new( name: h['name'],
+           **kwargs )
+  end
+
+
+  def initialize( name:,
+                  version: nil,
+                  version_created: nil,
+                  version_downloads: nil,
+                  homepage: nil,
+                  yanked: nil,
+                  runtime: [] )
+
+     @name              = name
+     @version           = version
+     @version_created   = version_created
+     @version_downloads = version_downloads
+     @homepage          = homepage
+     @yanked            = yanked
+     @runtime           = runtime
+  end
+
+  alias_method :runtime_dependencies, :runtime
+
+  ## always return false if not defined - why? why not?
+  def yanked?()   @yanked.nil?  ? false : @yanked;  end
+end  # class Gems
+
+
+end   ## module Gems

data/lib/gemverse/timeline.rb

@@ -0,0 +1,161 @@
+
+module Gems
+class Timeline
+
+  def initialize( versions )
+    ## sort by version created and name
+    @versions = versions.sort do |l,r|
+                  res =  r['created'] <=> l['created']
+                  res =  l['name']    <=> r['name']     if res == 0
+                  res =  r['version'] <=> l['version']  if res == 0
+                  res
+                end
+  end
+
+
+  def save( path )
+     buf = build
+     write_text( path, buf )
+  end
+
+
+  def build
+   ## step 1 - build document model - versions by year & week
+   ##                  and split into new & update
+   model = {}
+
+   @versions.each do |rec|
+
+     date = Date.strptime( rec['created'], '%Y-%m-%d' )
+     year = date.year.to_s
+     week = date.strftime('%V')  ## note: return zero-padded string e.g. 01, 02, etc.
+
+     by_year = model[ year ] ||= {}
+     by_week = by_year[ week ] ||= { 'new'     => [],
+                                     'updated' => [] }
+
+     if rec['count'].to_i == 1
+        by_week[ 'new' ] << rec
+     else
+        by_week[ 'updated' ] << rec
+     end
+   end
+
+   ## pp model
+
+   ## step 2 - build document
+
+   buf = String.new
+   buf << "# Timeline \n\n"
+
+   ## add breadcrumps for years
+   buf << model.keys.map do |year|
+                 "[#{year}](##{year})"
+          end.join( ' · ' )
+   buf << "\n\n"
+
+   ## add new gems by year
+   buf << "## New Gems By Year\n\n"
+
+   model.each do |year, by_week|
+    ## get totals for year
+    gems_new = by_week.values.reduce( [] ) do |acc,gems|
+      acc + gems['new']
+    end
+
+    buf << "**#{year}** - "
+
+    buf <<   if gems_new.size > 0
+                _build_gems_new( gems_new )
+             else
+               "Ø \n\n"
+             end
+   end
+
+
+   model.each do |year, by_week|
+    buf << "## #{year}\n\n"
+
+    by_week.each do |week, gems|
+      buf << "**Week #{week}**\n\n"
+
+      buf << _build_gems_new( gems['new'] )
+      buf << _build_gems_updated( gems['updated'] )
+    end
+   end
+
+   buf
+  end  # method build
+
+
+def _build_gem( gem )
+  date =  Date.strptime( gem['created'], '%Y-%m-%d' )
+
+  buf = String.new
+  buf << "[**"
+  buf << gem['name']
+  buf << "**]"
+  buf << "("
+  buf << "https://rubygems.org/gems/#{gem['name']}/versions/#{gem['version']}"
+  buf << " "
+  buf << %Q{"update no.#{gem['count']} @ #{date.strftime('%a %d %b %Y')}"}
+  buf << ") "
+  buf << gem['version']
+  buf
+end
+
+
+def _build_gems_new( gems )
+  buf = String.new
+  if gems.size > 0
+    buf << "**NEW!** - #{gems.size} Gem(s) - "
+    buf <<  gems.map do |gem| _build_gem( gem )
+                     end.join( ', ' )
+    buf << "\n\n"
+  end
+  buf
+end
+
+def _build_gems_updated( gems )
+  buf = String.new
+  if gems.size > 0
+    buf << "#{gems.size} Update(s)  - "
+    buf <<  gems.map do |gem| _build_gem( gem )
+                     end.join( ', ' )
+   buf << "\n\n"
+  end
+  buf
+end
+
+
+
+
+
+  def export( path )
+
+    headers = ['week',
+           'name',
+           'version',
+           'count',
+           'created',
+           'downloads']
+   recs = []
+   @versions.each_with_index do |h,i|
+
+     date = Date.strptime( h['created'], '%Y-%m-%d' )
+
+     rec = [
+        date.strftime('%Y/%V'),
+        h['name'],
+        h['version'],
+        h['count'],
+        date.strftime( '%Y-%m-%d' ),  # add (%a) for (Sun), (Mon) or such - why? why not?
+        h['downloads'],
+      ]
+      recs << rec
+    end
+
+    write_csv( path, [headers]+recs )
+  end
+end  ## class Timeline
+end  ## module Gems

data/lib/gemverse.rb

@@ -0,0 +1,95 @@
+require 'cocos'
+
+
+## move to cocos - upstream - why? why not?
+def write_csv( path, recs )
+  buf = String.new
+  buf << recs[0].join( ', ' )
+  buf << "\n"
+  recs[1..-1].each do |values|
+    buf << values.join( ', ' )
+    buf << "\n"
+  end
+  write_text( path, buf )
+end
+
+
+
+
+
+
+require_relative 'gemverse/api'
+require_relative 'gemverse/gems'
+require_relative 'gemverse/cache'
+
+require_relative 'gemverse/timeline'  ## timeline report
+
+
+
+module Gems
+
+###
+##  "high-level" finders
+
+def self.find_by( owner: )  ## todo/check: use
+
+  rows = API.gems_by( owner )
+  # pp data
+  puts "  #{rows.size} record(s) founds"
+
+  ## write "raw respone" to cache for debugging
+  write_json( "./cache/gems_#{owner}.json", rows )
+
+
+
+  recs = []
+  rows.each do |row|
+     recs <<  Gem.create( row )
+  end
+
+  ## sort by
+  ##  1) (latest) version created and
+  ##  2) name
+   recs = recs.sort do |l,r|
+     res =  r.version_created <=> l.version_created
+     res =  l.name            <=> r.name       if res == 0
+     res
+   end
+
+  GemDataset.new( recs )
+end
+
+
+
+def self.read_csv( path )
+  ## note: requires Kernel::  otherwise stackoverflow endlessly calling read_csv
+  rows = Kernel::read_csv( path )
+  puts "  #{rows.size} record(s) founds"
+
+  recs = []
+  rows.each do |row|
+     kwargs = {
+      version: row['version'].empty? ? nil : row['version'],
+      version_downloads: row['version_downloads'].empty? ? nil : row['version_downloads'].to_i,
+      version_created:  row['version_created'].empty? ? nil : Date.strptime( row['version_created'], '%Y-%m-%d' ),
+      homepage:  row['homepage'].empty? ? nil : row['homepage'],
+      runtime:   row['dependencies'].empty? ? [] : row['dependencies'].split('|').map {|dep| dep.strip },
+     }
+     recs <<  Gem.new( name: row['name'], **kwargs )
+  end
+
+  ## sort by
+  ##  1) (latest) version created and
+  ##  2) name
+   recs = recs.sort do |l,r|
+     res =  r.version_created <=> l.version_created
+     res =  l.name            <=> r.name     if res == 0
+     res
+   end
+
+  GemDataset.new( recs )
+end
+
+
+end  # module Gems
+

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: gemverse
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-01-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cocos
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.23'
+description: gemverse gem - gem universe incl. rubygems API V1 wrapper lite; gem version
+  cache, gem timeline reports, 'n' more
+email: opensport@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+files:
+- CHANGELOG.md
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/gemverse.rb
+- lib/gemverse/api.rb
+- lib/gemverse/cache.rb
+- lib/gemverse/gems.rb
+- lib/gemverse/timeline.rb
+homepage: https://github.com/rubycocos/git
+licenses:
+- Public Domain
+metadata: {}
+post_install_message:
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: gemverse gem - gem universe incl. rubygems API V1 wrapper lite; gem version
+  cache, gem timeline reports, 'n' more
+test_files: []