spandx-rubygems 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 62bf7a46ea4f16ce6bb797c50d880d7950178b74303b8dbcc13e933b9d258efc
+  data.tar.gz: b165132d0109bdcb52d7616415e95b24edd32852553dc799d4814dae038269d3
+SHA512:
+  metadata.gz: 9e02ef991ebbb605f38add61ca6bd5a01d641ec5e0f05dc9c1197f408ff007c8bbe115370e0098ebb016f1e27df8e647e675bc2bc84b20e8375ee6bb6ae0fd16
+  data.tar.gz: fba05641039ab64f880bf62a4a22bd9a8279428e0b479978f9f0615cf64b254ff78c733a41ab04d9c36aa235fc37365cfd5ac201629ae3c0ec04d6db41acb8e1

data/.gitattributes

@@ -0,0 +1 @@
+*.index filter=lfs diff=lfs merge=lfs -text

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.rspec_status
+*.tar
+db

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,70 @@
+require:
+  - rubocop-rspec
+
+AllCops:
+  Exclude:
+    - 'pkg/**/*'
+    - 'spec/fixtures/**/*'
+  TargetRubyVersion: 2.7
+
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+Layout/ParameterAlignment:
+  Enabled: true
+  EnforcedStyle: with_fixed_indentation
+  IndentationWidth: 2
+
+Layout/EndOfLine:
+  EnforcedStyle: lf
+
+Layout/FirstArrayElementIndentation:
+  EnforcedStyle: consistent
+
+Layout/MultilineMethodCallIndentation:
+  Enabled: true
+  EnforcedStyle: indented
+
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - '*.gemspec'
+    - 'Rakefile'
+    - 'spec/**/*.rb'
+
+Metrics/ModuleLength:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Layout/LineLength:
+  Exclude:
+    - 'spec/**/*.rb'
+  IgnoredPatterns:
+    - '^#*'
+
+Naming/RescuedExceptionsVariableName:
+  PreferredName: error
+
+Style/Documentation:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: 'single_quotes'
+
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Max: 80
+
+RSpec/NamedSubject:
+  Enabled: false
+
+RSpec/FilePath:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in spandx-rubygems.gemspec
+gemspec
+
+gem 'rake', '~> 12.0'
+gem 'rspec', '~> 3.0'

data/Gemfile.lock

@@ -0,0 +1,83 @@
+PATH
+  remote: .
+  specs:
+    spandx-rubygems (0.1.0)
+      msgpack (~> 1.3)
+      net-hippie (~> 0.3)
+      nokogiri (~> 1.10)
+      pg (~> 1.2)
+      thor
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.0)
+    bundler-audit (0.6.1)
+      bundler (>= 1.2.0, < 3)
+      thor (~> 0.18)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    hashdiff (1.0.0)
+    jaro_winkler (1.5.4)
+    mini_portile2 (2.4.0)
+    msgpack (1.3.3)
+    net-hippie (0.3.2)
+    nokogiri (1.10.7)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.19.1)
+    parser (2.7.0.2)
+      ast (~> 2.4.0)
+    pg (1.2.2)
+    public_suffix (4.0.3)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    rubocop (0.79.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-rspec (1.37.1)
+      rubocop (>= 0.68.1)
+    ruby-progressbar (1.10.1)
+    safe_yaml (1.0.5)
+    thor (0.20.3)
+    unicode-display_width (1.6.1)
+    vcr (5.1.0)
+    webmock (3.8.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler-audit (~> 0.6)
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rubocop (~> 0.52)
+  rubocop-rspec (~> 1.22)
+  spandx-rubygems!
+  vcr (~> 5.1)
+  webmock (~> 3.8)
+
+BUNDLED WITH
+   2.1.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 mo khan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# spandx/rubygems
+
+This project generates an index of every gem on rubygems.org and it's equivalent software licenses.
+This index is meant to speed up license scanning for the `spandx` CLI.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'spandx-rubygems'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install spandx-rubygems
+
+## Usage
+
+This will download each weekly backup from https://rubygems.org/pages/data,
+restore it to a local postgresql database mounted in `./db/data`,
+then generate a in memory hash to finally flush it to `rubygems.index` using
+message pack.
+
+You will need to have ruby and postgres installed.
+
+```bash
+$ ./bin/db-setup
+$ ./bin/server
+$ ruby ./exe/spandx-rubygems
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mokhan/spandx-rubygems. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/spandx-rubygems/blob/master/CODE_OF_CONDUCT.md).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/audit/task'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new(:rubocop)
+Bundler::Audit::Task.new
+
+task lint: [:rubocop, 'bundle:audit']
+task default: :spec

data/bin/cibuild

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+echo [$(date "+%H:%M:%S")] "==> Started at…"
+
+# GC customizations
+export RUBY_GC_MALLOC_LIMIT=79000000
+export RUBY_GC_HEAP_INIT_SLOTS=800000
+export RUBY_HEAP_FREE_MIN=100000
+export RUBY_HEAP_SLOTS_INCREMENT=400000
+export RUBY_HEAP_SLOTS_GROWTH_FACTOR=1
+
+ruby -v
+gem install bundler --conservative -v '~> 2.0'
+git submodule update --init
+bin/test
+bin/lint

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'spandx/rubygems'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/db-setup

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+DATA_DIR=$PWD/db/data
+rm -fr $DATA_DIR
+initdb -D $DATA_DIR

data/bin/lint

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+[ -z "$DEBUG" ] || set -x
+
+echo [$(date "+%H:%M:%S")] "==> Running setup…"
+bin/setup
+
+echo [$(date "+%H:%M:%S")] "==> Running linters…"
+bundle exec rake lint

data/bin/load

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+echo "Adding hstore extension"
+bin/sql -q -c "CREATE EXTENSION IF NOT EXISTS hstore;"
+
+echo "Dropping tables that we're going to load in"
+bin/sql -q -c "DROP TABLE IF EXISTS dependencies, gem_downloads, linksets, rubygems, versions CASCADE;"
+
+latest_url="$1"
+curl --progress-bar "${latest_url}" > tmp/public_postgresql.tar
+
+# Extract the single PostgresSQL.sql.gz file from the tar file, pass it through gunzip
+# and load it as quietly as possible into the database
+tar xOf tmp/public_postgresql.tar public_postgresql/databases/PostgreSQL.sql.gz | \
+  gunzip -c | \
+  bin/sql
+
+rm tmp/public_postgresql.tar
+echo "Done."

data/bin/server

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+export PGDATA=$PWD/db/data
+mkdir -p tmp/sockets
+postgres -F -D $PGDATA -k $PWD/tmp/sockets

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/shipit

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+[ -z "$DEBUG" ] || set -x
+
+bin/cibuild
+bundle exec rake release

data/bin/sql

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+psql -h $PWD/tmp/sockets postgres "$@"

data/bin/test

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+[ -z "$DEBUG" ] || set -x
+
+echo [$(date "+%H:%M:%S")] "==> Running setup…"
+bin/setup
+
+echo [$(date "+%H:%M:%S")] "==> Running tests…"
+bundle exec rake spec

data/exe/spandx-rubygems

@@ -0,0 +1,6 @@
+#!/bin/env ruby
+# frozen_string_literal: true
+
+require 'spandx/rubygems/cli'
+
+Spandx::Rubygems::CLI.start

data/lib/spandx/rubygems.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'msgpack'
+require 'net/hippie'
+require 'nokogiri'
+require 'pg'
+require 'yaml'
+require 'zlib'
+
+require 'spandx/rubygems/backup'
+require 'spandx/rubygems/backups'
+require 'spandx/rubygems/data_file'
+require 'spandx/rubygems/index'
+require 'spandx/rubygems/version'
+
+module Spandx
+  module Rubygems
+    class Error < StandardError; end
+
+    class << self
+      def root
+        Pathname.new(File.dirname(__FILE__)).join('rubygems')
+      end
+    end
+  end
+end