type_fusion 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7fd6c97b4b4bb6eeecea4ff23e108c24a7564ac3d85e2231352824cd27bf160e
-  data.tar.gz: a438ca7330e335d8d1922c967a24ef46a23403d6825e0e7a4740aafa6e93dcf6
+  metadata.gz: ba90818fd8e50f24a1b0049c0a8076a22b2e168a0fcafff53fb425b858b6323a
+  data.tar.gz: '011790f72f4d7e17fe12a8a1443f71e4bbdc42d4308f02c8110d5b07a3e91825'
 SHA512:
-  metadata.gz: 48d5ea9c6b6c52e8fe44bd8b1ea295f96ff5e48e737985b7c75d21b5c8379d2934b8dfe6e0fa41acfc85aa4104427a7e86354ebed07823d81e37c85a5060d7e3
-  data.tar.gz: 54d0eae75e1b154a249ddf5eecf6facf7694d3fc43d0ea3e3c022e7225a0510f7e2d58758fbbba885f6b45c16f35236653054e3e1d9ea199b6dfb16696b88fa3
+  metadata.gz: 23e4e3de6448d1f0f1a982ed90d73fec09b74d10b89e037bb6c477c24c8576bb42087a3dc112b612e41f81029e364472ebe498fd83258dc7da4f368c44866a26
+  data.tar.gz: 933b4a204122078dcc8a939daa35aae1648cb73ca06ec08231aa982be790adabc8075725ac715782873c84e8c657a3b6f12db9759b1fe32008dbde3557f70148

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 ## [Unreleased]
 
+## [0.0.5] - 2023-08-25
+
+- Introduce `TypeFusion::Processor` for processing samples
+- Add support for running TypeFusion in the test environment with Minitest
+- Rescue and log when job failed to enqueue
+- Add Rake task for manually processing enqueued jobs
+
+## [0.0.4] - 2023-08-16
+
+- Implement sample collection by sending samples to gem.sh endpoint
+- Introduce Litejob to enqueue samples to be collected async
+- Updates to `SampleCall`
+  - Split up `gem_and_version` to `gem` and `version` in `SampleCall`
+  - Also collect `application_name` and `type_fusion_version` with `SampleCall`
+  - Change `Tracepoint` API from `:call` to `:return` to also collect `return_value` in `SampleCall`
+- Introduce `endpoint` and `application_name` configs
+
 ## [0.0.3] - 2023-08-13
 
 - Introduce `TypeFusion::Middleware` to allow type-sampling in Rack-powered apps

data/Gemfile

@@ -6,6 +6,4 @@ gemspec
 
 gem "minitest", "~> 5.0"
 gem "rake", "~> 13.0"
-gem "rubocop", "~> 1.21"
-
-gem "litestack", github: "marcoroth/litestack"
+gem "rubocop", "~> 1.56"

data/Gemfile.lock

@@ -1,54 +1,69 @@
-GIT
-  remote: https://github.com/marcoroth/litestack.git
-  revision: 1911d5ef35f474425fb4e17a11a7d40deb4008a8
-  specs:
-    litestack (0.2.6)
-      erubi
-      hanami-router
-      oj
-      rack
-      sqlite3
-      tilt
-
 PATH
   remote: .
   specs:
-    type_fusion (0.0.3)
-      litestack
+    type_fusion (0.0.5)
+      lhc (~> 15.2)
+      litejob (~> 0.2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (7.0.7.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
-    erubi (1.12.0)
-    hanami-router (0.6.2)
-      hanami-utils (~> 0.7)
-      http_router (~> 0.11)
-    hanami-utils (0.9.2)
-    http_router (0.11.2)
-      rack (>= 1.0.0)
-      url_mount (~> 0.2.1)
+    base64 (0.1.1)
+    concurrent-ruby (1.2.2)
+    ethon (0.16.0)
+      ffi (>= 1.15.0)
+    ffi (1.15.5)
+    i18n (1.14.1)
+      concurrent-ruby (~> 1.0)
     json (2.6.3)
+    language_server-protocol (3.17.0.3)
+    lhc (15.2.1)
+      activesupport (>= 5.2)
+      addressable
+      local_uri
+      typhoeus (>= 0.11)
+    litedb (0.2.1)
+      litescheduler (>= 0.2.0)
+      sqlite3 (>= 1.5.0)
+    litejob (0.2.3)
+      litequeue (>= 0.2.1)
+      litescheduler (>= 0.2.1)
+    litequeue (0.2.1)
+      litedb (>= 0.2.1)
+    litescheduler (0.2.1)
+    local_uri (1.2.0)
+      activesupport
+      rack
     minitest (5.19.0)
-    oj (3.15.1)
     parallel (1.23.0)
     parser (3.2.2.3)
       ast (~> 2.4.1)
       racc
+    public_suffix (5.0.3)
     racc (1.7.1)
     rack (3.0.8)
     rainbow (3.1.1)
     rake (13.0.6)
     regexp_parser (2.8.1)
     rexml (3.2.6)
-    rubocop (1.48.1)
+    rubocop (1.56.1)
+      base64 (~> 0.1.1)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.3)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.26.0, < 2.0)
+      rubocop-ast (>= 1.28.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.29.0)
@@ -56,20 +71,20 @@ GEM
     ruby-progressbar (1.13.0)
     sqlite3 (1.6.3-x86_64-darwin)
     sqlite3 (1.6.3-x86_64-linux)
-    tilt (2.2.0)
+    typhoeus (1.4.0)
+      ethon (>= 0.9.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (2.4.2)
-    url_mount (0.2.1)
-      rack
 
 PLATFORMS
   x86_64-darwin-22
   x86_64-linux
 
 DEPENDENCIES
-  litestack!
   minitest (~> 5.0)
   rake (~> 13.0)
-  rubocop (~> 1.21)
+  rubocop (~> 1.56)
   type_fusion!
 
 BUNDLED WITH

data/README.md

@@ -27,6 +27,24 @@ use TypeFusion::Middleware
 Adding the gem to your applications Gemfile will automatically setup `type_fusion`.
 
 
+#### Minitest
+
+If you only want to run TypeFusion in your test environment you can require `type_fusion/minitest` in your `test_helper.rb`:
+
+```diff
+# test/test_helper.rb
+
+ENV["RAILS_ENV"] ||= "test"
+require_relative "../config/environment"
+require "rails/test_help"
+
++require "type_fusion/minitest"
+
+class ActiveSupport::TestCase
+  # ...
+end
+```
+
 ## Configuration
 
 Setup `TypeFusion` in an initializer
@@ -38,6 +56,27 @@ require "type_fusion"
 
 TypeFusion.config do |config|
 
+  # === application_name
+  #
+  # Set application_name to a string which is used to know where the samples
+  # came from. Set application_name to an empty string if you wish to not
+  # send the application name alongside the samples.
+  #
+  # Default: "TypeFusion"
+  # Default when using Rails: Rails.application.class.module_parent_name
+  #
+  # config.application_name = "YourApplication"
+
+
+  # === endpoint
+  #
+  # Set endpoint to an URL where TypeFusion should send the samples to.
+  #
+  # Default: "https://gem.sh/api/v1/types/samples"
+  #
+  # config.endpoint = "https://your-domain.com/api/v1/types/samples"
+
+
   # === type_sample_request
   #
   # Set type_sample_request to a lambda which resolves to true/false
@@ -54,13 +93,16 @@ TypeFusion.config do |config|
   # to true/false to check if a tracepoint_path should be sampled
   # or not.
   #
-  # This can be useful when you only want to sample method calls for
+  # This can be useful when you want to only sample method calls for
   # certain gems or want to exclude a gem from being sampled.
   #
   # Example:
   # config.type_sample_tracepoint_path = ->(tracepoint_path) {
-  #   # only sample calls for the Nokogiri gem
-  #   tracepoint_path.include?("nokogiri")
+  #   return false if tracepoint_path.include?("activerecord")
+  #   return false if tracepoint_path.include?("sprockets")
+  #   return false if tracepoint_path.include?("some-private-gem")
+  #
+  #   true
   # }
   #
   # Default: ->(tracepoint_path) { true }
@@ -110,19 +152,20 @@ TypeFusion::Sampler.instance.samples
 TypeFusion::Sampler.instance.samples.first
 
 # => #<struct TypeFusion::SampleCall
-#      gem_and_version="nokogiri-1.15.4-x86_64-darwin",
+#      gem_name="nokogiri"
+#      gem_version="1.15.4-x86_64-darwin",
 #      receiver="Nokogiri",
 #      method_name=:parse,
-#      location=[
-#        "/Users/marcoroth/.anyenv/envs/rbenv/versions/3.2.2/lib/ruby/gems/3.2.0/gems/nokogiri-1.15.4-x86_64-darwin/lib/nokogiri.rb",
-#        43
-#      ],
+#      application_name="TypeFusion",
+#      location="/Users/marcoroth/.anyenv/envs/rbenv/versions/3.2.2/lib/ruby/gems/3.2.0/gems/nokogiri-1.15.4-x86_64-darwin/lib/nokogiri.rb:43",
+#      type_fusion_version="0.0.4",
 #      parameters=[
 #        [:string, :req, String],
 #        [:url, :opt, NilClass],
 #        [:encoding, :opt, NilClass],
 #        [:options, :opt, NilClass]
-#      ]
+#      ],
+#      return_value="Nokogiri::XML::Document"
 #     >
 ```
 

data/lib/tasks/type_fusion.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+namespace :type_fusion do
+  desc "Process enqueued type samples"
+  task :process do
+    TypeFusion.processor.process!
+  end
+end

data/lib/type_fusion/config.rb

@@ -11,17 +11,20 @@ module TypeFusion
 
       @config
     end
+    alias configure config
   end
 
   class Config
     include Singleton
 
-    attr_accessor :type_sample_call_rate, :type_sample_request, :type_sample_tracepoint_path
+    attr_accessor :type_sample_call_rate, :type_sample_request, :type_sample_tracepoint_path, :endpoint, :application_name
 
     def initialize
       @type_sample_call_rate = 0.001
       @type_sample_request = ->(_env) { [true, false, false, false].sample }
       @type_sample_tracepoint_path = ->(_tracepoint_path) { true }
+      @endpoint = "https://gem.sh/api/v1/types/samples"
+      @application_name = "TypeFusion"
     end
 
     def type_sample_request?(env)

data/lib/type_fusion/litejob.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "litejob"
+
+# Litequeue.configure do |config|
+#   config.path = "db/type_fusion/queue.sqlite3"
+# end
+
+Litejob.configure do |config|
+  config.logger = Class.new do
+    def self.info(*args)
+    end
+  end
+end

data/lib/type_fusion/minitest.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+TypeFusion.start
+
+Minitest.after_run do
+  TypeFusion.stop
+  TypeFusion.processor.process!
+end

data/lib/type_fusion/processor.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module TypeFusion
+  class Processor
+    def initialize(amount = 4)
+      @amount = amount
+      @servers = []
+    end
+
+    def should_run?
+      enqueued_samples.positive?
+    end
+
+    def enqueued_samples
+      Litequeue.instance.count
+    end
+
+    def stop!
+      @servers.each do |server|
+        server.instance_variable_get(:@scheduler).context.kill
+      end
+      @servers = []
+    end
+
+    def start!
+      Signal.trap("INT") do
+        puts "\n[TypeFusion] Stop processing of TypeFusion samples..."
+        stop!
+      end
+
+      @amount.times do
+        @servers << Litejob::Server.new
+      end
+    end
+
+    def process!
+      puts "[TypeFusion] Start processing of #{enqueued_samples} samples..."
+
+      start!
+
+      while should_run?
+        puts "[TypeFusion] Remaining samples: #{Litequeue.instance.count}"
+        sleep 1
+      end
+
+      puts "[TypeFusion] Finished!"
+
+      puts "[TypeFusion] Congratulations! You just contributed samples to the following gems and helped making the Ruby ecosystem better for the whole community! Thank you!\n"
+      puts "- #{gems.join("\n- ")}"
+
+      stop!
+    end
+
+    private
+
+    def gems
+      TypeFusion::Sampler.instance.samples.map(&:gem_name).uniq.sort
+    end
+  end
+end

data/lib/type_fusion/rack/middleware.rb

@@ -6,6 +6,7 @@ module TypeFusion
   class Middleware
     def initialize(app)
       @app = app
+      TypeFusion.start_processing
     end
 
     def call(env)

data/lib/type_fusion/rails/railtie.rb

@@ -5,6 +5,8 @@ module TypeFusion
     initializer "type_fusion.middleware" do |app|
       require "type_fusion/rack/middleware"
 
+      TypeFusion.config.application_name = app.class.module_parent_name
+
       app.config.middleware.use TypeFusion::Middleware
     end
   end

data/lib/type_fusion/sample_call.rb

@@ -3,9 +3,13 @@
 require "json"
 
 module TypeFusion
-  SampleCall = Struct.new(:gem_and_version, :receiver, :method_name, :location, :parameters) do
+  SampleCall = Struct.new(:gem_name, :gem_version, :receiver, :method_name, :application_name, :location, :type_fusion_version, :parameters, :return_value) do
     def to_s
       JSON.pretty_generate(to_h)
     end
+
+    def inspect
+      "#<TypeFusion::SampleCall receiver=#{receiver.inspect} method_name=#{method_name.inspect} gem_name=#{gem_name.inspect}>"
+    end
   end
 end

data/lib/type_fusion/sample_job.rb

@@ -1,15 +1,17 @@
 # frozen_string_literal: true
 
-require "litestack"
+require "lhc"
 
 module TypeFusion
   class SampleJob
     include Litejob
 
-    self.queue = :default
+    queue_as :default
 
     def perform(sample)
-      puts sample.inspect
+      LHC.json.post(TypeFusion.config.endpoint, body: { sample: JSON.parse(sample) })
+    rescue StandardError => e
+      puts e.inspect
     end
   end
 end

data/lib/type_fusion/sampler.rb

@@ -21,7 +21,7 @@ module TypeFusion
     end
 
     def trace
-      @trace ||= TracePoint.trace(:call) do |tracepoint|
+      @trace ||= TracePoint.trace(:return) do |tracepoint|
         if sample?(tracepoint.path)
           receiver = begin
             tracepoint.binding.receiver.name
@@ -30,21 +30,32 @@ module TypeFusion
           end
 
           method_name = tracepoint.method_id
-          location = tracepoint.binding.source_location
-          gem_and_version = location.first.gsub(gem_path, "").split("/").first
-          args = tracepoint.parameters.map(&:reverse).to_h
+          location = tracepoint.binding.source_location.join(":")
+          gem_and_version = location.gsub(gem_path, "").split("/").first
+          gem, version = gem_and_version_from(gem_and_version)
+          args = tracepoint.parameters.to_h(&:reverse)
           parameters = extract_parameters(args, tracepoint.binding)
+          return_value = type_for_object(tracepoint.return_value)
 
           sample = SampleCall.new(
-            gem_and_version: gem_and_version,
+            gem_name: gem,
+            gem_version: version,
             receiver: receiver,
             method_name: method_name,
             location: location,
+            type_fusion_version: VERSION,
             parameters: parameters,
+            application_name: TypeFusion.config.application_name,
+            return_value: return_value,
           )
 
           samples << sample
-          SampleJob.perform_async(sample)
+
+          begin
+            SampleJob.perform_async(sample)
+          rescue StandardError => e
+            puts "[TypeFusion] Couldn't enqueue sample: #{e.inspect}"
+          end
         end
       end.tap(&:disable)
     end
@@ -81,6 +92,31 @@ module TypeFusion
       end
     end
 
+    def gem_and_version_from(gem_and_version)
+      return [] if gem_and_version.nil?
+
+      splits = gem_and_version.split("-")
+
+      if splits.length == 1
+        [splits.first, nil]
+      elsif splits.length == 2
+        splits
+      else
+        *name, version = splits
+
+        # TODO: there must be a better way to do this
+        if ["darwin", "java", "linux", "ucrt", "mingw32"].include?(version)
+          amount = (version == "ucrt") ? 3 : 2
+
+          version = [*name.pop(amount), version].join("-")
+        end
+
+        gem = name.join("-")
+
+        [gem, version]
+      end
+    end
+
     def extract_parameters(args, binding)
       args.map do |name, kind|
         variable = name.to_s.gsub("*", "").gsub("&", "").to_sym

data/lib/type_fusion/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TypeFusion
-  VERSION = "0.0.3"
+  VERSION = "0.0.5"
 end

data/lib/type_fusion.rb

@@ -2,9 +2,11 @@
 
 require_relative "type_fusion/version"
 require_relative "type_fusion/config"
+require_relative "type_fusion/litejob"
 require_relative "type_fusion/sample_call"
 require_relative "type_fusion/sample_job"
 require_relative "type_fusion/sampler"
+require_relative "type_fusion/processor"
 
 module TypeFusion
   class << self
@@ -22,6 +24,16 @@ module TypeFusion
       Sampler.instance.trace.disable
     end
 
+    def start_processing
+      puts "[TypeFusion] Start processing..."
+      processor.start!
+    end
+
+    def stop_processing
+      puts "[TypeFusion] Stop processing..."
+      processor.stop!
+    end
+
     def with_sampling
       start
 
@@ -29,6 +41,10 @@ module TypeFusion
 
       stop
     end
+
+    def processor
+      @processor ||= TypeFusion::Processor.new(4)
+    end
   end
 end
 

data/test.rb

@@ -16,4 +16,4 @@ TypeFusion.with_sampling do
 end
 
 puts TypeFusion::Sampler.instance.samples
-puts TypeFusion::Sampler.instance.to_s
+puts TypeFusion::Sampler.instance

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: type_fusion
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Marco Roth
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-13 00:00:00.000000000 Z
+date: 2023-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: litestack
+  name: lhc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '15.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '15.2'
+- !ruby/object:Gem::Dependency
+  name: litejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
 description: Community-contributed sample data for Ruby types
 email:
 - marco.roth@intergga.ch
@@ -39,8 +53,12 @@ files:
 - Gemfile.lock
 - README.md
 - Rakefile
+- lib/tasks/type_fusion.rake
 - lib/type_fusion.rb
 - lib/type_fusion/config.rb
+- lib/type_fusion/litejob.rb
+- lib/type_fusion/minitest.rb
+- lib/type_fusion/processor.rb
 - lib/type_fusion/rack/middleware.rb
 - lib/type_fusion/rails/railtie.rb
 - lib/type_fusion/sample_call.rb