typeprof 0.9.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c3bf54ec6fee3b06397893d2f10f1fae8e7af82b8f1640a4ddd305c33922ff7
-  data.tar.gz: 944d40a75789b4805e53ca70e169fe68fcd4dbca7f24bc4ccc779964085e980d
+  metadata.gz: 239b17dbe8d61c729346c5dde966f6abff491951c88bc33c57c4c34cc99dc497
+  data.tar.gz: 2696c83796a6af5c498cca59a3d98b7faccaf187fe60c0057ac3f052622009ff
 SHA512:
-  metadata.gz: 726cdbfcda694f189a96f3f35088cbcbd9a3064e79e3838f7758d7bc393bf8591c0559db1c17993f971f9ae2ee4ca3b4c09f5c0ef6d6675752b00b6948cdfe62
-  data.tar.gz: b2b7353855183df391d8bc78c247eb7de09c1b65b2753789eb1e1fb0208275d10fec6d2d95745f36771de30af5e32281476100a0654db6f7cf87c8e442b36a65
+  metadata.gz: 23a7a3313dadd2d0c63a5d83a3afd112893134a049861e6a9e11cf5097aaa78c19494326b6a07b44cf4d93f5dec5a2c893b91fe2c4f5571726229d1e8dc13229
+  data.tar.gz: 3273f1bb24290af32db23aaae8cafa84905d4d2d576b5ab96178130fe843b73e4365827b60d47a022a2812621f7fe09a3801c32c205c9805516873ed9ac88659

data/.github/workflows/main.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: [2.7.1, head]
+        ruby-version: [2.7, 3.0, head]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2

data/Gemfile.lock

@@ -1,17 +1,17 @@
 PATH
   remote: .
   specs:
-    typeprof (0.9.0)
-      rbs (>= 0.20.0)
+    typeprof (0.12.0)
+      rbs (>= 1.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     coverage-helpers (1.0.0)
-    docile (1.3.2)
+    docile (1.3.4)
     power_assert (1.2.0)
     rake (13.0.1)
-    rbs (0.20.1)
+    rbs (1.0.3)
     simplecov (0.20.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -24,6 +24,7 @@ GEM
 
 PLATFORMS
   ruby
+  x86_64-linux
 
 DEPENDENCIES
   coverage-helpers
@@ -35,4 +36,4 @@ DEPENDENCIES
   typeprof!
 
 BUNDLED WITH
-   2.2.0.rc.1
+   2.2.3

data/doc/demo.md

@@ -39,7 +39,7 @@ class Object
 end
 ```
 
-Yoy can [try this analysis online](https://mame.github.io/typeprof-playground/#rb=def+hello_message%28user%29%0A++%22The+name+is+%22+%2B+user.name%0Aend%0A%0Adef+type_error_demo%28user%29%0A++%22The+age+is+%22+%2B+user.age%0Aend%0A%0Auser+%3D+User.new%28name%3A+%22John%22%2C+age%3A+20%29%0A%0Ahello_message%28user%29%0Atype_error_demo%28user%29&rbs=class+User%0A++attr_reader+name%3A+String%0A++attr_reader+age%3A+Integer%0A%0A++def+initialize%3A+%28name%3A+String%2C+age%3A+Integer%29+-%3E+void%0Aend).
+You can [try this analysis online](https://mame.github.io/typeprof-playground/#rb=def+hello_message%28user%29%0A++%22The+name+is+%22+%2B+user.name%0Aend%0A%0Adef+type_error_demo%28user%29%0A++%22The+age+is+%22+%2B+user.age%0Aend%0A%0Auser+%3D+User.new%28name%3A+%22John%22%2C+age%3A+20%29%0A%0Ahello_message%28user%29%0Atype_error_demo%28user%29&rbs=class+User%0A++attr_reader+name%3A+String%0A++attr_reader+age%3A+Integer%0A%0A++def+initialize%3A+%28name%3A+String%2C+age%3A+Integer%29+-%3E+void%0Aend).
 
 ## A simple demo to generate the signature prototype of "User" class
 
@@ -190,7 +190,7 @@ p [:a, :b, :c] #=> [:a, :b, :c]
 # A Hash is a "type-to-type" map
 h = { "int" => 1, "float" => 1.0 }
 p h        #=> {String=>Float | Integer}
-p h["int"] #=> Float | Intger
+p h["int"] #=> Float | Integer
 
 # Symbol-key hashes (a.k.a. records) can have distinct types for each key as Symbols are concrete
 h = { int: 1, float: 1.0 }

data/doc/todo.md

@@ -0,0 +1,133 @@
+# TypeProf milestones and TODOs
+
+## Big milestones
+
+### Rails support
+
+There are many known issues for the analysis of typical Ruby programs including Rails apps.
+
+* The main difficulty is that they use many language extensions like `ActiveSupport`.
+  Some features (for example, `blank?` and `Time.now + 1.day`) are trivial to support,
+  but others (for example, `ActiveSupport::Concern` and Zeitwerk) will require special support.
+* The other difficulty is that they heavily use meta-programming features like `ActiveRecord`.
+  It dynamically defines some methods based on external data (such as DB schema) from the code.
+
+Currently, projects called [`gem_rbs`](https://github.com/ruby/gem_rbs) and [`rbs_rails`](https://github.com/pocke/rbs_rails) are in progress.
+The former provides several RBS files for some major gems including Rails.
+The latter is a tool to generate RBS prototype of a target Rails application by introspection (executing it and monitoring DB schema, etc).
+TypeProf can use their results to improve analysis precision and performance.
+
+What we need to do:
+
+* Experimentally apply TypeProf to some Rails programs and identify problems
+* Make TypeProf able to work together with `rbs_rails` for supporting trivial core extensions and `ActiveRecord`.
+* Implement special support for some fundamental language extensions of Rails like `ActiveSupport::Concern`.
+  (It would be best if TypeProf has a plugin system and if we can factor out the special support as a plugin for Rails.)
+
+### Error detection and diagnosis feature
+
+At present, TypeProf focuses on generation of RBS prototype from no-type-annotated Ruby code.
+However, it is possible for TypeProf to report possible errors found during the analysis.
+In fact, an option `-v` experimentally shows possible errors found.
+There are some reasons why it is disabled by default:
+
+* (1) There are too many false positives.
+* (2) Some kind of error reporting is not implemented yet.
+* (3) Some reported errors are difficult for a user to understand.
+
+For (1), we will research how we can avoid false positives to support typical Ruby coding patterns as much as possible.
+The primary way is to improve the analysis precision, e.g., enhancing flow-sensitive analysis.
+If the S/N ratio of an error type is too low, we need to consider to suppress the kind of reports.
+Also, we may try allowing users to guide TypeProf to analyze their program well.
+(The simplest way is to write inline type casts in the code, but we need to find more Ruby/RBS way.)
+We may also explore a "TypeProf-friendly coding style" which TypeProf can analyze well.
+(In principle, the plainer code is, the better TypeProf can analyze.)
+
+For (2), currently, TypeProf checks the argument types of a call to a method whose type signature is declared in RBS.
+However, it does not check the return type yet. Redefinition of constants should be warned too.
+We will survey what errors and warnings TypeProf can print, and evaluate the S/N ratio of each report.
+
+For (3), since TypeProf uses whole program analysis, an error may be reported at a very different place from its root bug.
+Thus, if TypeProf shows a possible type error, a diagnosis feature is needed to answer why TypeProf thinks that the error may occur.
+TypeProf has already implemented a very primitive diagnosis feature, `Kernel#p`, to check what type an expression has.
+Another idea is to create a pseudo backtrace why TypeProf thought the possible type error may occur.
+We should consider this feature with LSP support.
+
+### Performance improvement
+
+Currently, TypeProf is painfully slow. Even if a target application is small.
+
+The main reason is that TypeProf analyzes not only the application code but also library code:
+if an application requires `"foo"`, TypeProf actually loads `foo.rb` even from a gem,
+and furthermore, if `foo.rb` requires `"bar"`, it loads `bar.rb` recursively.
+
+RBS will help to stop this cascade;
+when an application requires `"foo"`, TypeProf loads `sig/foo.rbs` instead of `foo.rb` if the `foo` gem contains both.
+Such a RBS file is optional for TypeProf but required for Steep.
+So, we think many gems will eventually equip their RBS declarations.
+
+That being said, we should continue to improve the analysis performance of TypeProf. We have some ideas.
+
+* Unfortunately, TypeProf often analyzes one method more than once when it accepts multiple types.
+  As TypeProf squashes the argument types to a union, this duplicated analysis is not necessarily needed.
+  But when TypeProf first analyzes a method, it is difficult to determine if the method will accept another type in further analysis.
+  So, we need good heuristics to guess whether a method accepts multiple types or not, and if so, delay its analysis.
+* Currently, TypeProf executes the bytecode instructions step by step.
+  This requires creating an environment object after each instruction, which is very heavy.
+  Many environment creations can be omitted by executing each basic block instead of each instruction.
+  (Basic block execution will also make flow-sensitive analysis easier.)
+* The slowest calculation in TypeProf is to create an instance of a Type class.
+  The creation uses memoization; TypeProf keeps all Type instances created so far, and reuses them if already exist.
+  However, it is very heavy to check if an instance already exists or not.
+  (Currently, it is very simply implemented by a big Hash table.)
+  We've already improved the memoization routine several times but looks like it is still the No.1 bottleneck.
+  We need to investigate and try improving more.
+* TypeProf heavily uses Hash objects (including above) mainly to represent a set.
+  A union of sets is done by `Hash#merge`, which takes O(n).
+  A more lightweight data structure may make TypeProf faster.
+  (But clever structure often has a big constant term, so we need to evaluate the performance carefully.)
+* Reusing an old analysis and incrementally updating it will bring a super big improvement.
+  This would be especially helpful for LSP support, so we need to tackle it after the analysis approach is mature.
+
+### Language Server Protocol (LSP) support
+
+In the future, we want TypeProf to serve as a language server to show the result in IDE in real-time.
+However, the current analysis approach is too slow for IDE. So we need to improve the performance first.
+
+Even if TypeProf becomes fast enough, its approach has a fundamental problem.
+Since TypeProf uses whole program analysis, one edit may cause a cascade of propagation:
+if a user write `foo(42)`, an Integer is propagated to a method `foo`,
+and if `foo` passes its argument to a method `bar`, it is propagated to `bar`, ...
+So, a breakthrough for LSP may be still needed, e.g, limiting the propagation range in real-time analysis,
+assuming that a type interface of module boundary is fixed, etc.
+
+## Relatively smaller TODOs
+
+* Support more RBS features
+  * TypeProf does not deal with some RBS types well yet.
+    * For example, the `instance` type is handled as `untyped.
+    * The `self` type is handled well only when it is used as a return type.
+    * Using a value of the `void` type should be warned appropriately.
+  * RBS's `interface` is supported just like a module (i.e., `include _Foo` is explicitly required in RBS),
+    but it should be checked structually (i.e., it should be determined as a method set.)
+  * The variance of type parameters is currently ignored.
+
+* Support more Ruby features
+  * Some meta-programming features like `Class.new`, `Object#method`, etc.
+    * It is possible to support `Class.new` by per-allocation-site approach:
+      e.g., In TypeProf, `A = Class.new; B = Class.new` will create two classes, but `2.times { Class.new }` will create one class.
+  * The analysis precision can be improved more for some Ruby features like pattern matching, keyword arguments, etc.
+    * For example, `foo(*args, k:1)` is currently compiled as if it is `foo(*(args + [{ :k => 1 }]))` into Ruby bytecode.
+      This mixes the keyword arguments to a rest array, and makes it difficult for TypeProf to track the keyword arguments.
+  * Support Enumerator as an Array-type container.
+  * Support `Module#protect` (but RBS does not yet).
+  * More heuristics may help such as `==` returns a bool regardless to its receiver and argument types.
+
+* Make TypeProf more useful as a tool
+  * Currently, TypeProf provides only the analysis engine and a minimal set of features.
+  * The analysis result would be useful not only to generate RBS prototype
+    but also identifying the source location of a method definition, listing callsites of a method,
+    searching a method call by its argument types, etc.
+  * Sometimes, TypeProf prints very big union type, such as `Integer | Float | Complex | Rational | ...`.
+    Worse, the same big type is printed multiple times.
+    It may be useful to factor out such a long type by using type alias, for example.

data/lib/typeprof/analyzer.rb

@@ -222,6 +222,11 @@ module TypeProf
       Env.new(senv, @locals, @stack, @type_params)
     end
 
+    def replace_blk_ty(ty)
+      senv = StaticEnv.new(@static_env.recv_ty, ty, @static_env.mod_func, @static_env.pub_meth)
+      Env.new(senv, @locals, @stack, @type_params)
+    end
+
     def inspect
       "Env[#{ @static_env.inspect }, locals:#{ @locals.inspect }, stack:#{ @stack.inspect }, type_params:#{ (@type_params&.internal_hash).inspect }]"
     end
@@ -295,7 +300,10 @@ module TypeProf
       def initialize(kind, name, absolute_path)
         raise unless name.is_a?(Array)
         @kind = kind
-        @modules = { true => [], false => [] }
+        @modules = {
+          :before => { true => [], false => [] }, # before = include/extend
+          :after  => { true => [], false => [] }, # after = prepend
+        }
         @name = name
         @consts = {}
         @methods = {}
@@ -308,13 +316,13 @@ module TypeProf
       attr_reader :kind, :modules, :consts, :methods, :ivars, :cvars, :absolute_path
       attr_accessor :name, :klass_obj
 
-      def include_module(mod, type_args, singleton, absolute_path)
-        mod_, module_type_args, absolute_paths = @modules[singleton].find {|m,| m == mod }
+      def mix_module(kind, mod, type_args, singleton, absolute_path)
+        mod_, module_type_args, absolute_paths = @modules[kind][singleton].find {|m,| m == mod }
         if mod_
-          raise "inconsistent include/extend type args in RBS?" if module_type_args != type_args && type_args != [] && type_args != nil
+          raise "inconsistent #{ kind == :after ? "include/extend" : "prepend" } type args in RBS?" if module_type_args != type_args && type_args != [] && type_args != nil
         else
           absolute_paths = Utils::MutableSet.new
-          @modules[singleton].unshift([mod, type_args, absolute_paths])
+          @modules[kind][singleton].unshift([mod, type_args, absolute_paths])
         end
         absolute_paths << absolute_path
       end
@@ -331,10 +339,8 @@ module TypeProf
         @consts[name] = [ty, absolute_path]
       end
 
-      def adjust_substitution(singleton, mid, mthd, subst, direct, &blk)
-        mthds = @methods[[singleton, mid]]
-        yield subst, direct if mthds&.include?(mthd)
-        @modules[singleton].each do |mod_def, type_args,|
+      def adjust_substitution_for_module(mods, mid, mthd, subst, &blk)
+        mods.each do |mod_def, type_args,|
           if mod_def.klass_obj.type_params && type_args
             subst2 = {}
             mod_def.klass_obj.type_params.zip(type_args) do |(tyvar, *), tyarg|
@@ -346,13 +352,28 @@ module TypeProf
         end
       end
 
+      def adjust_substitution(singleton, mid, mthd, subst, direct, &blk)
+        adjust_substitution_for_module(@modules[:before][singleton], mid, mthd, subst, &blk)
+
+        mthds = @methods[[singleton, mid]]
+        yield subst, direct if mthds&.include?(mthd)
+
+        adjust_substitution_for_module(@modules[:after][singleton], mid, mthd, subst, &blk)
+      end
+
       def search_method(singleton, mid, visited, &blk)
         # Currently, circular inclusion of modules is allowed
         return if visited[self]
         visited[self] = true
+
+        @modules[:before][singleton].each do |mod_def,|
+          mod_def.search_method(false, mid, visited, &blk)
+        end
+
         mthds = @methods[[singleton, mid]]
         yield mthds, @klass_obj, singleton if mthds
-        @modules[singleton].each do |mod_def,|
+
+        @modules[:after][singleton].each do |mod_def,|
           mod_def.search_method(false, mid, visited, &blk)
         end
       end
@@ -381,17 +402,17 @@ module TypeProf
       end
     end
 
-    def include_module(including_mod, included_mod, type_args, singleton, caller_ep)
-      return if included_mod == Type.any
+    def mix_module(kind, mixing_mod, mixed_mod, type_args, singleton, caller_ep)
+      return if mixed_mod == Type.any
 
-      including_mod = @class_defs[including_mod.idx]
-      included_mod.each_child do |included_mod|
-        if included_mod.is_a?(Type::Class)
-          included_mod = @class_defs[included_mod.idx]
-          if included_mod && included_mod.kind == :module
-            including_mod.include_module(included_mod, type_args, singleton, caller_ep ? caller_ep.ctx.iseq.absolute_path : nil)
+      mixing_mod = @class_defs[mixing_mod.idx]
+      mixed_mod.each_child do |mixed_mod|
+        if mixed_mod.is_a?(Type::Class)
+          mixed_mod = @class_defs[mixed_mod.idx]
+          if mixed_mod && mixed_mod.kind == :module
+            mixing_mod.mix_module(kind, mixed_mod, type_args, singleton, caller_ep ? caller_ep.ctx.iseq.absolute_path : nil)
           else
-            warn(caller_ep, "including something that is not a module")
+            warn(caller_ep, "attempted to #{ kind == :after ? "include/extend" : "prepend" } non-module; ignored")
           end
         end
       end
@@ -578,12 +599,12 @@ module TypeProf
       mdef
     end
 
-    def add_attr_method(klass, absolute_path, mid, ivar, kind)
+    def add_attr_method(klass, mid, ivar, kind, pub_meth, ep)
       if kind == :reader || kind == :accessor
-        add_method(klass, mid, false, AttrMethodDef.new(ivar, :reader, absolute_path))
+        add_method(klass, mid, false, AttrMethodDef.new(ivar, :reader, pub_meth, ep))
       end
       if kind == :writer || kind == :accessor
-        add_method(klass, :"#{ mid }=", false, AttrMethodDef.new(ivar, :writer, absolute_path))
+        add_method(klass, :"#{ mid }=", false, AttrMethodDef.new(ivar, :writer, pub_meth, ep))
       end
     end
 
@@ -595,22 +616,22 @@ module TypeProf
       add_method(klass, mid, true, ISeqMethodDef.new(iseq, cref, outer_ep, pub_meth))
     end
 
-    def set_custom_method(klass, mid, impl)
-      set_method(klass, mid, false, CustomMethodDef.new(impl))
+    def set_custom_method(klass, mid, impl, pub_meth = true)
+      set_method(klass, mid, false, CustomMethodDef.new(impl, pub_meth))
     end
 
-    def set_singleton_custom_method(klass, mid, impl)
-      set_method(klass, mid, true, CustomMethodDef.new(impl))
+    def set_singleton_custom_method(klass, mid, impl, pub_meth = true)
+      set_method(klass, mid, true, CustomMethodDef.new(impl, pub_meth))
     end
 
-    def alias_method(klass, singleton, new, old)
+    def alias_method(klass, singleton, alias_mid, orig_mid, ep)
       if klass == Type.any
         self
       else
-        mdefs = get_method(klass, singleton, old)
+        mdefs = get_method(klass, singleton, orig_mid)
         if mdefs
           mdefs.each do |mdef|
-            @class_defs[klass.idx].add_method(new, singleton, mdef)
+            @class_defs[klass.idx].add_method(alias_mid, singleton, AliasMethodDef.new(orig_mid, mdef, ep))
           end
         end
       end
@@ -689,7 +710,9 @@ module TypeProf
         entry = @tbl[site] ||= Entry.new(false, {}, Type.bot, Utils::MutableSet.new)
         entry.read_continuations[ep] = ctn
         entry.absolute_paths << ep.ctx.iseq.absolute_path if ep.ctx.is_a?(Context)
-        ctn[entry.type, ep]
+        ty = entry.type
+        ty = Type.nil if ty == Type.bot
+        ctn[ty, ep]
       end
 
       def add_write!(site, ty, ep, scratch)
@@ -697,7 +720,7 @@ module TypeProf
         if ep
           if entry.rbs_declared
             unless Type.match?(ty, entry.type)
-              scratch.warn(ep, "inconsistent assignment to RBS-declared global variable")
+              scratch.warn(ep, "inconsistent assignment to RBS-declared variable")
               return
             end
           end
@@ -1050,7 +1073,7 @@ module TypeProf
         lead_tys = env.locals[0, lead_num].map {|ty| globalize_type(ty, env, ep) }
         opt_tys = opt.size > 1 ? env.locals[lead_num, opt.size - 1].map {|ty| globalize_type(ty, env, ep) } : []
         if rest_start # XXX:squash
-          ty = globalize_type(env.locals[lead_num + opt.size - 1], env, ep)
+          ty = globalize_type(env.locals[rest_start], env, ep)
           rest_ty = Type.bot
           ty.each_child_global do |ty|
             if ty.is_a?(Type::Array)
@@ -1085,6 +1108,7 @@ module TypeProf
           end
         end
         kw_rest_ty = globalize_type(env.locals[kw_rest], env, ep) if kw_rest
+        kw_rest_ty = nil if kw_rest_ty == Type.nil
         if block_start
           blk_ty = globalize_type(env.locals[block_start], env, ep)
         elsif iseq.type == :method
@@ -1264,7 +1288,7 @@ module TypeProf
           end
         end
         return
-      when :send_branch
+      when :getlocal_send_branch
         getlocal_operands, send_operands, branch_operands = operands
         env, recvs, mid, aargs = setup_actual_arguments(:method, send_operands, ep, env)
         recvs = Type.any if recvs == Type.bot
@@ -1292,6 +1316,31 @@ module TypeProf
           end
         end
         return
+      when :send_branch
+        send_operands, branch_operands = operands
+        env, recvs, mid, aargs = setup_actual_arguments(:method, send_operands, ep, env)
+        recvs = Type.any if recvs == Type.bot
+        recvs.each_child do |recv|
+          do_send(recv, mid, aargs, ep, env) do |ret_ty, ep, env|
+            env, ret_ty, = localize_type(ret_ty, env, ep)
+
+            branchtype, target, = branch_operands
+            # branchtype: :if or :unless or :nil
+            ep_then = ep.next
+            ep_else = ep.jump(target)
+
+            case ret_ty
+            when Type::Instance.new(Type::Builtin[:true])
+              merge_env(branchtype == :if ? ep_else : ep_then, env)
+            when Type::Instance.new(Type::Builtin[:false])
+              merge_env(branchtype == :if ? ep_then : ep_else, env)
+            else
+              merge_env(ep_then, env)
+              merge_env(ep_else, env)
+            end
+          end
+        end
+        return
       when :invokeblock
         env, recvs, mid, aargs = setup_actual_arguments(:block, operands, ep, env)
         blk = env.static_env.blk_ty
@@ -2104,7 +2153,7 @@ module TypeProf
 
       bsig ||= BlockSignature.new([], [], nil, Type.nil)
 
-      bsig = bsig.screen_name(self)#, block: true)
+      bsig = bsig.screen_name(nil, self)
       ret_ty = ret_ty.screen_name(self)
       ret_ty = (ret_ty.include?("|") ? "(#{ ret_ty })" : ret_ty) # XXX?
 
@@ -2131,7 +2180,7 @@ module TypeProf
         end
       end
 
-      farg_tys = farg_tys ? farg_tys.screen_name(self) : "(unknown)"
+      farg_tys = farg_tys ? farg_tys.screen_name(nil, self) : "(unknown)"
       ret_ty = ret_ty.screen_name(self)
       ret_ty = (ret_ty.include?("|") ? "(#{ ret_ty })" : ret_ty) # XXX?
 
@@ -2143,10 +2192,13 @@ module TypeProf
       farg_tys = @method_signatures[ctx]
       ret_ty = @return_values[ctx] || Type.bot
 
-      farg_tys = farg_tys.screen_name(self)
+      untyped = farg_tys.include_untyped?(self) || ret_ty.include_untyped?(self)
+
+      farg_tys = farg_tys.screen_name(ctx.iseq, self)
       ret_ty = ret_ty.screen_name(self)
       ret_ty = (ret_ty.include?("|") ? "(#{ ret_ty })" : ret_ty) # XXX?
-      "#{ (farg_tys.empty? ? "" : "#{ farg_tys } ") }-> #{ ret_ty }"
+
+      ["#{ (farg_tys.empty? ? "" : "#{ farg_tys } ") }-> #{ ret_ty }", untyped]
     end
   end
 end