skylight 1.0.0.beta4 → 1.0.0.beta5

data/lib/skylight/normalizers/active_record/sql.rb

@@ -1,4 +1,3 @@
-require "sql_lexer"
 require "json"
 
 module Skylight
@@ -41,24 +40,8 @@ module Skylight
         private
 
         def extract_binds(payload, precalculated)
-          case config[:sql_mode]
-          when 'rust'.freeze
-            extract_rust(payload)
-          when 'ruby'.freeze
-            extract_ruby(payload, precalculated)
-          else
-            raise "Unrecognized sql_mode: #{config.sql_mode}"
-          end
-        end
-
-        def extract_rust(payload)
           Skylight.lex_sql(payload[:sql])
         end
-
-        def extract_ruby(payload, precalculated)
-          name, title, _ = SqlLexer::Lexer.bindify(payload[:sql], precalculated, true)
-          [ name, title ]
-        end
       end
     end
   end

data/lib/skylight/normalizers/grape/endpoint.rb

@@ -12,7 +12,7 @@ module Skylight
 
           def get_method(endpoint)
             method = endpoint.options[:method].first
-            method << "..." if endpoint.options[:method].length > 1
+            method = "#{method}..." if endpoint.options[:method].length > 1
             method
           end
 

data/lib/skylight/probes/action_controller.rb

@@ -8,11 +8,18 @@ module Skylight
             alias append_info_to_payload_without_sk append_info_to_payload
             def append_info_to_payload(payload)
               append_info_to_payload_without_sk(payload)
+              if respond_to?(:rendered_format)
+                rendered_mime = rendered_format
+              else
+                format = lookup_context.formats.first
+                rendered_mime = Mime[format.to_sym] if format
+              end
+              payload[:rendered_format] = rendered_mime.try(:ref)
               payload[:variant] = request.respond_to?(:variant) ? request.variant : nil
             end
           end
 
-          #if RAILS_VERSION < 4.2.1
+          if Gem::Version.new(Rails.version) < Gem::Version.new('4.2.1')
             # Backport https://github.com/rails/rails/pull/17978
             ::ActionController::Instrumentation.class_eval do
               def process_action(*args)
@@ -37,7 +44,7 @@ module Skylight
                   end
                 end
               end
-            #end
+            end
           end
         end
       end

data/lib/skylight/probes/sequel.rb

@@ -5,11 +5,14 @@ module Skylight
       class Probe
         def install
           require 'sequel/database/logging'
-          ::Sequel::Database.class_eval do
-            alias log_yield_without_sk log_yield
 
-            def log_yield(sql, args=nil, &block)
-              log_yield_without_sk(sql, *args) do
+          method_name = ::Sequel::Database.method_defined?(:log_connection_yield) ? 'log_connection_yield' : 'log_yield'
+
+          ::Sequel::Database.class_eval <<-end_eval
+            alias #{method_name}_without_sk #{method_name}
+
+            def #{method_name}(sql, *args, &block)
+              #{method_name}_without_sk(sql, *args) do
                 ::ActiveSupport::Notifications.instrument(
                   "sql.sequel",
                   sql: sql,
@@ -20,7 +23,7 @@ module Skylight
                 end
               end
             end
-          end
+          end_eval
         end
       end
     end

data/lib/skylight/probes/sinatra.rb

@@ -8,7 +8,9 @@ module Skylight
             alias compile_without_sk! compile!
 
             def compile!(verb, path, *args, &block)
-              compile_without_sk!(verb, path, *args, &block).tap do |_, _, _, wrapper|
+              compile_without_sk!(verb, path, *args, &block).tap do |_, _, keys_or_wrapper, wrapper|
+                wrapper ||= keys_or_wrapper
+
                 # Deal with the situation where the path is a regex, and the default behavior
                 # of Ruby stringification produces an unreadable mess
                 if path.is_a?(Regexp)

data/lib/skylight/railtie.rb

@@ -26,13 +26,11 @@ module Skylight
       if activate?
         if config
           begin
-            config.validate!
-
             if Instrumenter.start!(config)
               app.middleware.insert 0, Middleware, config: config
               Rails.logger.info "[SKYLIGHT] [#{Skylight::VERSION}] Skylight agent enabled"
             else
-              Rails.logger.info "[SKYLIGHT] [#{Skylight::VERSION}] Unable to start"
+              Rails.logger.info "[SKYLIGHT] [#{Skylight::VERSION}] Unable to start, see the Skylight logs for more details"
             end
           rescue ConfigError => e
             Rails.logger.error "[SKYLIGHT] [#{Skylight::VERSION}] #{e.message}; disabling Skylight agent"
@@ -88,7 +86,7 @@ module Skylight
         # Configure the log file destination
         if log_file = app.config.skylight.log_file
           config['log_file'] = log_file
-        elsif !config.key?('log_file')
+        elsif !config.key?('log_file') && !config.on_heroku?
           config['log_file'] = File.join(Rails.root, 'log/skylight.log')
         end
 

data/lib/skylight/subscriber.rb

@@ -86,8 +86,11 @@ module Skylight
 
       while curr = trace.notifications.pop
         if curr.name == name
-          normalize_after(trace, curr.span, name, payload)
-          trace.done(curr.span) if curr.span
+          begin
+            normalize_after(trace, curr.span, name, payload)
+          ensure
+            trace.done(curr.span) if curr.span
+          end
           return
         end
       end

data/lib/skylight/util/logging.rb

@@ -57,6 +57,7 @@ module Skylight
       end
 
       def error(msg, *args)
+        raise sprintf(msg, *args) if ENV['SKYLIGHT_RAISE_ON_ERROR']
         log :error, msg, *args
       end
 
@@ -69,8 +70,13 @@ module Skylight
       alias fmt       sprintf
 
       def log(level, msg, *args)
-        return unless respond_to?(:config)
-        return unless c = config
+        c = if respond_to?(:config)
+          config
+        elsif self.is_a?(Config)
+          self
+        end
+
+        return unless c
 
         if logger = c.logger
           return unless logger.respond_to?(level)

data/lib/skylight/util/native_ext_fetcher.rb

@@ -4,6 +4,7 @@ require 'net/http'
 require 'fileutils'
 require 'digest/sha2'
 require 'skylight/util/ssl'
+require 'skylight/util/proxy'
 
 # Used from extconf.rb
 module Skylight
@@ -92,9 +93,9 @@ module Skylight
               when :success
                 log "successfully downloaded native ext; out=#{out}"
                 return extra
-              else
+              when :redirect
                 log "fetching native ext; uri=#{uri}; redirected=#{res}"
-                uri = res
+                uri = extra
 
                 next
               end
@@ -118,7 +119,7 @@ module Skylight
       end
 
       def http_get(host, port, use_ssl, path, out)
-        if http_proxy = ENV['HTTP_PROXY'] || ENV['http_proxy']
+        if http_proxy = Proxy.detect_url(ENV)
           log "connecting with proxy: #{http_proxy}"
           uri = URI.parse(http_proxy)
           p_host, p_port = uri.host, uri.port

data/lib/skylight/util/proxy.rb

@@ -0,0 +1,12 @@
+module Skylight
+  module Util
+    module Proxy
+      def self.detect_url(env)
+        if u = env['HTTP_PROXY'] || env['http_proxy']
+          u = "http://#{u}" unless u =~ %r[://]
+          u
+        end
+      end
+    end
+  end
+end

data/lib/skylight/version.rb

@@ -1,4 +1,4 @@
 module Skylight
-  VERSION = '1.0.0-beta4'
+  VERSION = '1.0.0-beta5'
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skylight
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta4
+  version: 1.0.0.beta5
 platform: ruby
 authors:
 - Tilde, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -47,6 +47,8 @@ files:
 - lib/skylight.rb
 - lib/skylight/api.rb
 - lib/skylight/cli.rb
+- lib/skylight/cli/doctor.rb
+- lib/skylight/cli/helpers.rb
 - lib/skylight/compat.rb
 - lib/skylight/config.rb
 - lib/skylight/core.rb
@@ -114,6 +116,7 @@ files:
 - lib/skylight/util/multi_io.rb
 - lib/skylight/util/native_ext_fetcher.rb
 - lib/skylight/util/platform.rb
+- lib/skylight/util/proxy.rb
 - lib/skylight/util/ssl.rb
 - lib/skylight/vendor/active_support/notifications.rb
 - lib/skylight/vendor/active_support/notifications/fanout.rb
@@ -163,10 +166,6 @@ files:
 - lib/skylight/vendor/thread_safe/synchronized_cache_backend.rb
 - lib/skylight/version.rb
 - lib/skylight/vm/gc.rb
-- lib/sql_lexer.rb
-- lib/sql_lexer/lexer.rb
-- lib/sql_lexer/string_scanner.rb
-- lib/sql_lexer/version.rb
 homepage: http://www.skylight.io
 licenses: []
 metadata: {}
@@ -186,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Skylight is a smart profiler for Rails apps

data/lib/sql_lexer.rb

@@ -1,6 +0,0 @@
-require "sql_lexer/version"
-require "sql_lexer/string_scanner"
-require "sql_lexer/lexer"
-
-module SqlLexer
-end

data/lib/sql_lexer/lexer.rb

@@ -1,579 +0,0 @@
-# encoding: utf-8
-
-require "strscan"
-
-module SqlLexer
-  class Lexer
-    # SQL identifiers and key words must begin with a letter (a-z, but also
-    # letters with diacritical marks and non-Latin letters) or an underscore
-    # (_). Subsequent characters in an identifier or key word can be letters,
-    # underscores, digits (0-9), or dollar signs ($). Note that dollar signs
-    # are not allowed in identifiers according to the letter of the SQL
-    # standard, so their use might render applications less portable. The SQL
-    # standard will not define a key word that contains digits or starts or
-    # ends with an underscore, so identifiers of this form are safe against
-    # possible conflict with future extensions of the standard.
-    StartID       = %q<\p{Alpha}_>
-    PartID        = %q<\p{Alnum}_$>
-    OpPart        = %q<\+|\-(?!-)|\*|/(?!\*)|\<|\>|=|~|!|@|#|%|\^|&|\||\?|\.|,|\(|\)>
-    WS            = %q< \t\r\n>
-    OptWS         = %Q<[#{WS}]*>
-    End           = %Q<;|$>
-
-    InOp          = %Q<IN(?=#{OptWS}\\()>
-    ArrayOp       = %q<ARRAY(?=\[)>
-    ColonColonOp  = %Q<::(?=[#{StartID}])>
-    ArrayIndexOp  = %q<\\[(?:\-?\d+(?::\-?\d+)?|NULL)\\]>
-    SpecialOps    = %Q<#{InOp}(?=[#{WS}])|#{ColonColonOp}|#{ArrayOp}|#{ArrayIndexOp}>
-
-    StartQuotedID = %Q<">
-    StartTickedID = %Q<`>
-    StartString   = %Q<[a-zA-Z]?'>
-    StartDigit    = %q<[\p{Digit}\.]>
-
-    StartSelect   = %Q<SELECT(?=(?:[#{WS}]|#{OpPart}))>
-
-    # Binds that are also IDs do not need to be included here, since AfterOp (which uses StartBind)
-    # also checks for StartAnyId
-    StartBind     = %Q<#{StartString}|#{StartDigit}|#{SpecialOps}>
-
-    StartNonBind  = %Q<#{StartQuotedID}|#{StartTickedID}|\\$(?=\\p{Digit})>
-    TableNext     = %Q<(#{OptWS}((?=#{StartQuotedID})|(?=#{StartTickedID}))|[#{WS}]+(?=[#{StartID}]))>
-    StartAnyId    = %Q<"#{StartID}>
-    Placeholder   = %q<\$\p{Digit}+>
-
-    AfterID       = %Q<[#{WS};#{StartNonBind}]|(?:#{OpPart})|(?:#{ColonColonOp})|(?:#{ArrayIndexOp})|$>
-    ID            = %Q<[#{StartID}][#{PartID}]*(?=#{AfterID})>
-    AfterOp       = %Q<[#{WS}]|[#{StartAnyId}]|[#{StartBind}]|(#{StartNonBind})|;|$>
-    Op            = %Q<(?:#{OpPart})+(?=#{AfterOp})>
-    QuotedID      = %Q<#{StartQuotedID}(?:[^"]|"")*">
-    TickedID      = %Q<#{StartTickedID}(?:[^`]|``)*`>
-    NonBind       = %Q<#{ID}|#{Op}|#{QuotedID}|#{TickedID}|#{Placeholder}>
-    Type          = %Q<[#{StartID}][#{PartID}]*(?:\\(\d+\\)|\\[\\])?(?=#{AfterID})>
-    QuotedTable   = %Q<#{TickedID}|#{QuotedID}>
-
-    StringBody    = %q{(?:''|(?<!\x5C)(?:\x5C\x5C)*\x5C'|[^'])*}
-    String        = %Q<#{StartString}#{StringBody}'>
-
-    Digits        = %q<\p{Digit}+>
-    OptDigits     = %q<\p{Digit}*>
-    Exponent      = %Q<e[+\-]?#{Digits}>
-    OptExponent   = %Q<(?:#{Exponent})?>
-    HeadDecimal   = %Q<#{Digits}\\.#{OptDigits}#{OptExponent}>
-    TailDecimal   = %Q<#{OptDigits}\\.#{Digits}#{OptExponent}>
-    ExpDecimal    = %Q<#{Digits}#{Exponent}>
-
-    Number        = %Q<#{HeadDecimal}|#{TailDecimal}|#{ExpDecimal}|#{Digits}>
-
-    Literals      = %Q<(?:NULL|TRUE|FALSE)(?=(?:[#{WS}]|#{OpPart}|#{End}))>
-
-    TkWS          = %r<[#{WS}]+>u
-    TkOptWS       = %r<[#{WS}]*>u
-    TkOp          = %r<[#{OpPart}]>u
-    TkComment     = %r<^#{OptWS}--.*$>u
-    TkBlockCommentStart = %r</\*>u
-    TkBlockCommentEnd   = %r<\*/>u
-    TkPlaceholder = %r<#{Placeholder}>u
-    TkNonBind     = %r<#{NonBind}>u
-    TkType        = %r<#{Type}>u
-    TkQuotedTable = %r<#{QuotedTable}>iu
-    TkUpdateTable = %r<UPDATE#{TableNext}>iu
-    TkInsertTable = %r<INSERT[#{WS}]+INTO#{TableNext}>iu
-    TkDeleteTable = %r<DELETE[#{WS}]+FROM#{TableNext}>iu
-    TkFromTable   = %r<FROM#{TableNext}>iu
-    TkID          = %r<#{ID}>u
-    TkEnd         = %r<;?[#{WS}]*>u
-    TkBind        = %r<#{String}|#{Number}|#{Literals}>u
-    TkIn          = %r<#{InOp}>iu
-    TkColonColon  = %r<#{ColonColonOp}>u
-    TkArray       = %r<#{ArrayOp}>iu
-    TkArrayIndex  = %r<#{ArrayIndexOp}>iu
-    TkSpecialOp   = %r<#{SpecialOps}>iu
-    TkStartSelect = %r<#{StartSelect}>iu
-    TkStartSubquery = %r<\(#{OptWS}#{StartSelect}>iu
-    TkCloseParen  = %r<#{OptWS}\)>u
-
-    STATE_HANDLERS = {
-      begin:       :process_begin,
-      first_token: :process_first_token,
-      tokens:      :process_tokens,
-      bind:        :process_bind,
-      non_bind:    :process_non_bind,
-      placeholder: :process_placeholder,
-      table_name:  :process_table_name,
-      end:         :process_end,
-      special:     :process_special,
-      subquery:    :process_subquery,
-      in:          :process_in,
-      array:       :process_array
-    }
-
-    def self.bindify(string, binds=nil, strip_comments=false)
-      scanner = instance(string)
-      scanner.process(binds, strip_comments)
-      [scanner.title, scanner.output, scanner.binds]
-    end
-
-    attr_reader :output, :binds, :title
-
-    def self.pooled_value(name, default)
-      key = :"__skylight_sql_#{name}"
-
-      singleton_class.class_eval do
-        define_method(name) do
-          value = Thread.current[key] ||= default.dup
-          value.clear
-          value
-        end
-      end
-
-      __send__(name)
-    end
-
-    SCANNER_KEY = :__skylight_sql_scanner
-    LEXER_KEY   = :__skylight_sql_lexer
-
-    def self.scanner(string='')
-      scanner = Thread.current[SCANNER_KEY] ||= StringScanner.new('')
-      scanner.string = string
-      scanner
-    end
-
-    def self.instance(string)
-      lexer = Thread.current[LEXER_KEY] ||= new
-      lexer.init(string)
-      lexer
-    end
-
-    pooled_value :binds, []
-    pooled_value :table, "*" * 20
-
-    SPACE = " ".freeze
-
-    DEBUG = ENV["SKYLIGHT_SQL_DEBUG"]
-
-    def init(string)
-      @state   = :begin
-      @debug   = DEBUG
-      @binds   = self.class.binds
-      @table   = self.class.table
-      @title   = nil
-      @bind    = 0
-
-      self.string = string
-    end
-
-    def string=(value)
-      @input   = value
-
-      @scanner = self.class.scanner(value)
-
-      # intentionally allocates; we need to return a new
-      # string as part of this API
-      @output = value.dup
-    end
-
-    PLACEHOLDER = "?".freeze
-    UNKNOWN = "<unknown>".freeze
-
-    def process(binds, strip_comments)
-      process_comments(strip_comments)
-
-      @operation = nil
-      @provided_binds = binds
-
-      while @state
-        if @debug
-          p @state
-          p @scanner
-        end
-
-        __send__ STATE_HANDLERS[@state]
-      end
-
-      pos = 0
-      removed = 0
-
-      # intentionally allocates; the returned binds must
-      # be in a newly produced array
-      extracted_binds = Array.new(@binds.size / 2)
-
-      if @operation && !@table.empty?
-        @title = "" << @operation << SPACE << @table
-      end
-
-      while pos < @binds.size
-        if @binds[pos] == nil
-          extracted_binds[pos/2] = @binds[pos+1]
-        else
-          slice = @output[@binds[pos] - removed, @binds[pos+1]]
-          @output[@binds[pos] - removed, @binds[pos+1]] = PLACEHOLDER
-
-          extracted_binds[pos/2] = slice
-          removed += (@binds[pos+1] - 1)
-        end
-
-        pos += 2
-      end
-
-      @binds = extracted_binds
-      nil
-    end
-
-    def replace_comment(pos, length, strip)
-      if strip
-        # Dup the string if necessary so we aren't destructive to the original value
-        if @input == @original_input
-          @input = @input.dup
-          @scanner.string = @input
-        end
-
-        # Replace the comment with a space to ensure valid SQL
-        # Updating the input also updates the scanner
-        @input[pos, length] = SPACE
-        @output[pos, length] = SPACE
-
-        # Move back to start of removed string
-        @scanner.pos = pos
-      else
-        # Dup the string if necessary so we aren't destructive to the original value
-        @scanner.string = @input.dup if @scanner.string == @original_input
-
-        # Replace the comment with spaces
-        @scanner.string[pos, length] = SPACE*length
-      end
-    end
-
-    def process_comments(strip_comments)
-      @original_input = @input
-
-      # SQL treats comments as similar to whitespace
-      # Here we replace all comments with spaces of the same length so as to not affect binds
-
-      # Remove block comments
-      # SQL allows for nested comments so this takes a bit more work
-      while @scanner.skip_until(TkBlockCommentStart)
-        count = 1
-        pos = @scanner.charpos - 2
-
-        while true
-          # Determine whether we close the comment or start nesting
-          next_open  = @scanner.skip_until(TkBlockCommentStart)
-          @scanner.unscan if next_open
-          next_close = @scanner.skip_until(TkBlockCommentEnd)
-          @scanner.unscan if next_close
-
-          if next_open && next_open < next_close
-            # We're nesting
-            count += 1
-            @scanner.skip_until(TkBlockCommentStart)
-          else
-            # We're closing
-            count -= 1
-            @scanner.skip_until(TkBlockCommentEnd)
-          end
-
-          if count > 10_000
-            raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in block comments"
-          end
-
-          if count == 0
-            # We've closed all comments
-            length = @scanner.charpos - pos
-            replace_comment(pos, length, strip_comments)
-            break
-          end
-        end
-      end
-
-      @scanner.reset
-
-      # Remove single line comments
-      while @scanner.skip_until(TkComment)
-        pos = @scanner.charpos
-        len = @scanner.matched_size
-        replace_comment(pos-len, len, strip_comments)
-      end
-
-      @scanner.reset
-    end
-
-    def process_begin
-      @scanner.skip(TkOptWS)
-      @state = :first_token
-    end
-
-    OP_SELECT_FROM = "SELECT FROM".freeze
-    OP_UPDATE      = "UPDATE".freeze
-    OP_INSERT_INTO = "INSERT INTO".freeze
-    OP_DELETE_FROM = "DELETE FROM".freeze
-
-    def process_first_token
-      if @scanner.skip(TkStartSelect)
-        @operation = OP_SELECT_FROM
-        @state = :tokens
-      else
-        if @scanner.skip(TkUpdateTable)
-          @operation = OP_UPDATE
-        elsif @scanner.skip(TkInsertTable)
-          @operation = OP_INSERT_INTO
-        elsif @scanner.skip(TkDeleteTable)
-          @operation = OP_DELETE_FROM
-        end
-
-        @state = :table_name
-      end
-    end
-
-    def process_table_name
-      pos = @scanner.pos
-
-      if @scanner.skip(TkQuotedTable)
-        copy_substr(@input, @table, pos + 1, @scanner.pos - 1)
-      elsif @scanner.skip(TkID)
-        copy_substr(@input, @table, pos, @scanner.pos)
-      end
-
-      @state = :tokens
-    end
-
-    def process_tokens
-      @scanner.skip(TkOptWS)
-
-      if @operation == OP_SELECT_FROM && @table.empty? && @scanner.skip(TkFromTable)
-        @state = :table_name
-      elsif @scanner.match?(TkSpecialOp)
-        @state = :special
-      elsif @scanner.match?(TkBind)
-        @state = :bind
-      elsif @scanner.match?(TkPlaceholder)
-        @state = :placeholder
-      elsif @scanner.match?(TkNonBind)
-        @state = :non_bind
-      else
-        @state = :end
-      end
-    end
-
-    def process_placeholder
-      @scanner.skip(TkPlaceholder)
-
-      binds << nil
-
-      if !@provided_binds
-        @binds << UNKNOWN
-      elsif !@provided_binds[@bind]
-        @binds << UNKNOWN
-      else
-        @binds << @provided_binds[@bind]
-      end
-
-      @bind += 1
-
-      @state = :tokens
-    end
-
-    def process_special
-      if @scanner.skip(TkIn)
-        @scanner.skip(TkOptWS)
-        if @scanner.skip(TkStartSubquery)
-          @state = :subquery
-        else
-          @scanner.skip(/\(/u)
-          @state = :in
-        end
-      elsif @scanner.skip(TkArray)
-        @scanner.skip(/\[/u)
-        @state = :array
-      elsif @scanner.skip(TkColonColon)
-        if @scanner.skip(TkType)
-          @state = :tokens
-        else
-          @state = :end
-        end
-      elsif @scanner.skip(TkStartSubquery)
-        @state = :subquery
-      elsif @scanner.skip(TkArrayIndex)
-        @state = :tokens
-      end
-    end
-
-    def process_subquery
-      nest = 1
-      iterations = 0
-
-      while nest > 0
-        iterations += 1
-
-        if iterations > 10_000
-          raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in subquery"
-        end
-
-        if @debug
-          p @state
-          p @scanner
-          p nest
-          p @scanner.peek(1)
-        end
-
-        if @scanner.skip(TkStartSubquery)
-          nest += 1
-          @state = :tokens
-        elsif @scanner.skip(TkCloseParen)
-          nest -= 1
-          break if nest.zero?
-          @state = :tokens
-        elsif @state == :subquery
-          @state = :tokens
-        end
-
-        __send__ STATE_HANDLERS[@state]
-      end
-
-      @state = :tokens
-    end
-
-    def process_in
-      nest = 1
-      iterations = 0
-
-      @skip_binds = true
-      pos = @scanner.charpos - 1
-
-      while nest > 0
-        iterations += 1
-
-        if iterations > 10_000
-          raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in IN"
-        end
-
-        if @debug
-          p @state
-          p @scanner
-          p nest
-        end
-
-        if @scanner.skip(/\(/u)
-          nest += 1
-          process_tokens
-        elsif @scanner.skip(TkCloseParen)
-          nest -= 1
-          break if nest.zero?
-          process_tokens
-        else
-          process_tokens
-        end
-
-        __send__ STATE_HANDLERS[@state]
-      end
-
-      @binds << pos
-      @binds << @scanner.charpos - pos
-
-      @skip_binds = false
-
-      @state = :tokens
-    end
-
-    def process_array
-      nest = 1
-      iterations = 0
-
-      @skip_binds = true
-      pos = @scanner.charpos - 6
-
-      while nest > 0
-        iterations += 1
-
-        if iterations > 10_000
-          raise "The SQL '#{@scanner.string}' could not be parsed because of too many iterations in ARRAY"
-        end
-
-        if @debug
-          p "array loop"
-          p @state
-          p @scanner
-        end
-
-        if @scanner.skip(/\[/u)
-          nest += 1
-        elsif @scanner.skip(/\]/u)
-          nest -= 1
-
-          break if nest.zero?
-
-          # End of final nested array
-          next if @scanner.skip(/#{TkOptWS}(?=\])/u)
-        end
-
-        # A NULL array
-        next if @scanner.skip(/NULL/iu)
-
-        # Another nested array
-        next if @scanner.skip(/#{TkOptWS},#{TkOptWS}(?=\[)/u)
-
-        process_tokens
-
-        __send__ STATE_HANDLERS[@state]
-      end
-
-      @binds << pos
-      @binds << @scanner.charpos - pos
-
-      @skip_binds = false
-
-      @state = :tokens
-    end
-
-    def process_non_bind
-      @scanner.skip(TkNonBind)
-      @state = :tokens
-    end
-
-    def process_bind
-      pos = nil
-
-      unless @skip_binds
-        pos = @scanner.charpos
-      end
-
-      @scanner.skip(TkBind)
-
-      unless @skip_binds
-        @binds << pos
-        @binds << @scanner.charpos - pos
-      end
-
-      @state = :tokens
-    end
-
-    def process_end
-      if @scanner.skip(TkEnd)
-        if @scanner.eos?
-          @state = nil
-        else
-          process_tokens
-        end
-      end
-
-      # We didn't hit EOS and we couldn't process any tokens
-      if @state == :end
-        raise "The SQL '#{@scanner.string}' could not be parsed"
-      end
-    end
-
-  private
-    def copy_substr(source, target, start_pos, end_pos)
-      pos = start_pos
-
-      while pos < end_pos
-        target.concat source.getbyte(pos)
-        pos += 1
-      end
-    end
-
-    scanner
-    instance('')
-
-  end
-end