sequel 3.21.0 → 3.36.0

data/lib/sequel/extensions/pg_array.rb

@@ -0,0 +1,460 @@
+# The pg_array extension adds support for Sequel to handle
+# PostgreSQL's string and numeric array types.  It supports both
+# single and multi-dimensional arrays.  For integer and
+# float arrays, it uses a JSON-based parser which is usually written in C
+# and should be fairly fast.  For string and decimal arrays, it uses
+# a hand coded parser written in ruby that is unoptimized and probably
+# slow.
+#
+# This extension integrates with Sequel's native postgres adapter, so
+# that when array fields are retrieved, they are parsed and returned
+# as instances of Sequel::Postgres::PGArray subclasses.  PGArray is
+# a DelegateClass of Array, so it mostly acts like an array, but not
+# completely (is_a?(Array) is false).  If you want the actual array,
+# you can call PGArray#to_a.  This is done so that Sequel does not
+# treat a PGArray like an Array by default, which would cause issues.
+#
+# In addition to the parsers, this extension comes with literalizers
+# for PGArray using the standard Sequel literalization callbacks, so
+# they work with on all adapters.
+#
+# To turn an existing Array into a PGArray:
+#
+#   array.pg_array
+#
+# You can also provide a type, though it many cases it isn't necessary:
+#
+#   array.pg_array(:varchar) # or :int4, :"double precision", etc.
+#
+# So if you want to insert an array into an int4[] database column:
+#
+#   DB[:table].insert(:column=>[1, 2, 3].pg_array)
+#
+# If you would like to use PostgreSQL arrays in your model objects, you
+# probably want to modify the schema parsing/typecasting so that it
+# recognizes and correctly handles the arrays, which you can do by:
+#
+#   DB.extend Sequel::Postgres::PGArray::DatabaseMethods
+#
+# If you are not using the native postgres adapter, you probably
+# also want to use the typecast_on_load plugin in the model, and
+# set it to typecast the array column(s) on load.
+#
+# If you want an easy way to call PostgreSQL array functions and
+# operators, look into the pg_array_ops extension.
+#
+# This extension requires both the json and delegate libraries.
+#
+# == Additional License
+#
+# PGArray::Parser code was translated from Javascript code in the
+# node-postgres project and has the following additional license:
+# 
+# Copyright (c) 2010 Brian Carlson (brian.m.carlson@gmail.com)
+# 
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject
+# to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'delegate'
+require 'json'
+
+module Sequel
+  module Postgres
+    # Base class for the PostgreSQL array types.  Subclasses generally
+    # just deal with parsing, so instances manually created from arrays
+    # can use this class correctly.
+    class PGArray < DelegateClass(Array)
+      ARRAY = "ARRAY".freeze
+      DOUBLE_COLON = '::'.freeze
+      EMPTY_BRACKET = '[]'.freeze
+      OPEN_BRACKET = '['.freeze
+      CLOSE_BRACKET = ']'.freeze
+      COMMA = ','.freeze
+      BACKSLASH = '\\'.freeze
+      EMPTY_STRING = ''.freeze
+      OPEN_BRACE = '{'.freeze
+      CLOSE_BRACE = '}'.freeze
+      NULL = 'NULL'.freeze
+      QUOTE = '"'.freeze
+
+      module DatabaseMethods
+        ESCAPE_RE = /("|\\)/.freeze
+        ESCAPE_REPLACEMENT = '\\\\\1'.freeze
+
+        # Reset the conversion procs when extending the Database object, so
+        # it will pick up the array convertors.  This is only done for the native
+        # postgres adapter.
+        def self.extended(db)
+          db.reset_conversion_procs if db.respond_to?(:reset_conversion_procs)
+        end
+
+        # Handle arrays in bound variables
+        def bound_variable_arg(arg, conn)
+          case arg
+          when PGArray
+            bound_variable_array(arg.to_a)
+          when Array
+            bound_variable_array(arg)
+          else
+            super
+          end
+        end
+
+        # Make the column type detection deal with string and numeric array types.
+        def schema_column_type(db_type)
+          case db_type
+          when /\A(character( varying)?|text).*\[\]\z/io
+            :string_array
+          when /\A(integer|bigint|smallint)\[\]\z/io
+            :integer_array
+          when /\A(real|double precision)\[\]\z/io
+            :float_array
+          when /\Anumeric.*\[\]\z/io
+            :decimal_array
+          else
+            super
+          end
+        end
+
+        private
+
+        # Format arrays used in bound variables.
+        def bound_variable_array(a)
+          case a
+          when Array
+            "{#{a.map{|i| bound_variable_array(i)}.join(COMMA)}}"
+          when String
+            "\"#{a.gsub(ESCAPE_RE, ESCAPE_REPLACEMENT)}\""
+          else
+            literal(a)
+          end
+        end
+
+        # Given a value to typecast and the type of PGArray subclass:
+        # * If given a PGArray, just return the value (even if different subclass)
+        # * If given an Array, create a new instance of the subclass
+        # * If given a String, call the parser for the subclass with it.
+        def typecast_value_pg_array(value, klass)
+          case value
+          when PGArray
+            value
+          when Array
+            klass.new(value)
+          when String
+            klass.parse(value)
+          else
+            raise Sequel::InvalidValue, "invalid value for #{klass}: #{value.inspect}"
+          end
+        end
+
+        # Create typecast methods for the supported array types that
+        # delegate to typecast_value_pg_array with the related class.
+        %w'string integer float decimal'.each do |t|
+          class_eval("def typecast_value_#{t}_array(v) typecast_value_pg_array(v, PG#{t.capitalize}Array) end", __FILE__, __LINE__)
+        end
+      end
+
+      # PostgreSQL array parser that handles both text and numeric
+      # input.  Because PostgreSQL arrays can contain objects that
+      # can be literalized in any number of ways, it is not possible
+      # to make a fully generic parser.
+      #
+      # This parser is very simple and unoptimized, but should still
+      # be O(n) where n is the length of the input string.
+      class Parser
+        attr_reader :pos
+
+        # Set the source for the input, and any converter callable
+        # to call with objects to be created.  For nested parsers
+        # the source may contain text after the end current parse,
+        # which will be ignored.
+        def initialize(source, converter=nil)
+          @source = source
+          @source_length = source.length
+          @converter = converter 
+          @pos = -1
+          @entries = []
+          @recorded = ""
+          @dimension = 0
+        end
+
+        # Return 2 objects, whether the next character in the input
+        # was escaped with a backslash, and what the next character is.
+        def next_char
+          @pos += 1
+          if (c = @source[@pos..@pos]) == BACKSLASH
+            @pos += 1
+            [true, @source[@pos..@pos]]
+          else
+            [false, c]
+          end
+        end
+
+        # Add a new character to the buffer of recorded characters.
+        def record(c)
+          @recorded << c
+        end
+        
+        # Take the buffer of recorded characters and add it to the array
+        # of entries, and use a new buffer for recorded characters.
+        def new_entry(include_empty=false)
+          if !@recorded.empty? || include_empty
+            entry = @recorded
+            if entry == NULL && !include_empty
+              entry = nil
+            elsif @converter
+              entry = @converter.call(entry)
+            end
+            @entries.push(entry)
+            @recorded = ""
+          end
+        end
+
+        # Parse the input character by character, returning an array
+        # of parsed (and potentially converted) objects.
+        def parse(nested=false)
+          # quote sets whether we are inside of a quoted string.
+          quote = false
+          until @pos >= @source_length
+            escaped, char = next_char
+            if char == OPEN_BRACE && !quote
+              @dimension += 1
+              if (@dimension > 1)
+                # Multi-dimensional array encounter, use a subparser
+                # to parse the next level down.
+                subparser = self.class.new(@source[@pos..-1], @converter)
+                @entries.push(subparser.parse(true))
+                @pos += subparser.pos - 1
+              end
+            elsif char == CLOSE_BRACE && !quote
+              @dimension -= 1
+              if (@dimension == 0)
+                new_entry
+                # Exit early if inside a subparser, since the
+                # text after parsing the current level should be
+                # ignored as it is handled by the parent parser.
+                return @entries if nested
+              end
+            elsif char == QUOTE && !escaped
+              # If already inside the quoted string, this is the
+              # ending quote, so add the entry.  Otherwise, this
+              # is the opening quote, so set the quote flag.
+              new_entry(true) if quote
+              quote = !quote
+            elsif char == COMMA && !quote
+              # If not inside a string and a comma occurs, it indicates
+              # the end of the entry, so add the entry.
+              new_entry
+            else
+              # Add the character to the recorded character buffer.
+              record(char)
+            end
+          end
+          raise Sequel::Error, "array dimensions not balanced" unless @dimension == 0
+          @entries
+        end
+      end
+
+      # Parse the string using the generalized parser, setting the type
+      # if given.
+      def self.parse(string, type=nil)
+        new(Parser.new(string, method(:convert_item)).parse, type)
+      end
+
+      # Return the item as-is by default, making conversion a no-op.
+      def self.convert_item(s)
+        s
+      end
+      private_class_method :convert_item
+
+      # The type of this array.  May be nil if no type was given. If a type
+      # is provided, the array is automatically casted to this type when
+      # literalizing.  This type is the underlying type, not the array type
+      # itself, so for an int4[] database type, it should be :int4 or 'int4'
+      attr_accessor :array_type
+
+      # Set the array to delegate to, and a database type.
+      def initialize(array, type=nil)
+        super(array)
+        self.array_type = type
+      end
+
+      # The delegated object is always an array.
+      alias to_a __getobj__
+
+      # Append the array SQL to the given sql string. 
+      # If the receiver has a type, add a cast to the
+      # database array type.
+      def sql_literal_append(ds, sql)
+        sql << ARRAY
+        _literal_append(sql, ds, to_a)
+        if at = array_type
+          sql << DOUBLE_COLON << at.to_s << EMPTY_BRACKET
+        end
+      end
+
+      private
+
+      # Recursive method that handles multi-dimensional
+      # arrays, surrounding each with [] and interspersing
+      # entries with ,.
+      def _literal_append(sql, ds, array)
+        sql << OPEN_BRACKET
+        comma = false
+        commas = COMMA
+        array.each do |i|
+          sql << commas if comma
+          if i.is_a?(Array)
+            _literal_append(sql, ds, i)
+          else
+            ds.literal_append(sql, i)
+          end
+          comma = true
+        end
+        sql << CLOSE_BRACKET
+      end
+    end
+
+    # PGArray subclass handling integer and float types, using a fast JSON
+    # parser.  Does not handle numeric/decimal types, since JSON does deal
+    # with arbitrary precision (see PGDecimalArray for that).
+    class PGNumericArray < PGArray
+      # Character conversion map mapping input strings to JSON replacements
+      SUBST = {'{'.freeze=>'['.freeze, '}'.freeze=>']'.freeze, 'NULL'.freeze=>'null'.freeze}
+
+      # Regular expression matching input strings to convert
+      SUBST_RE = %r[\{|\}|NULL].freeze
+
+      # Parse the input string by using a gsub to convert non-JSON characters to
+      # JSON, running it through a regular JSON parser, and the doing a recursive
+      # map over the output to make sure the entries are in the correct type (mostly,
+      # to make sure real/double precision database types are returned as float and
+      # not integer).
+      def self.parse(string, type=nil)
+        new(recursive_map(JSON.parse(string.gsub(SUBST_RE){|m| SUBST[m]})), type)
+      end
+
+      # Convert each item in the array to the correct type, handling multi-dimensional
+      # arrays.
+      def self.recursive_map(array)
+        array.map do |i|
+          if i.is_a?(Array)
+            recursive_map(i)
+          elsif i
+            convert_item(i)
+          end
+        end
+      end
+      private_class_method :recursive_map
+    end
+
+    # PGArray subclass for decimal/numeric types.  Uses the general
+    # parser as the JSON parser cannot handle arbitrary precision numbers.
+    class PGDecimalArray < PGArray
+      # Convert the item to a BigDecimal.
+      def self.convert_item(s)
+        BigDecimal.new(s.to_s)
+      end
+      private_class_method :convert_item
+
+      ARRAY_TYPE = 'decimal'.freeze
+
+      # Use the decimal type by default.
+      def array_type
+        super || ARRAY_TYPE
+      end
+    end
+
+    # PGArray subclass for handling real/double precision arrays.
+    class PGFloatArray < PGNumericArray
+      # Convert the item to a float.
+      def self.convert_item(s)
+        s.to_f
+      end
+      private_class_method :convert_item
+
+      ARRAY_TYPE = 'double precision'.freeze
+
+      # Use the double precision type by default.
+      def array_type
+        super || ARRAY_TYPE
+      end
+    end
+
+    # PGArray subclass for handling int2/int4/int8 arrays.
+    class PGIntegerArray < PGNumericArray
+      ARRAY_TYPE = 'int4'.freeze
+
+      # Use the int4 type by default.
+      def array_type
+        super || ARRAY_TYPE
+      end
+    end
+
+    # PGArray subclass for handling char/varchar/text arrays.
+    class PGStringArray < PGArray
+      CHAR = 'char'.freeze
+      VARCHAR = 'varchar'.freeze
+      TEXT = 'text'.freeze
+
+      # By default, use a text array.  If char is given without
+      # a size, use varchar instead, as otherwise Postgres assumes
+      # length of 1, which is likely to cause data loss.
+      def array_type
+        case (c = super)
+        when nil 
+          TEXT
+        when CHAR, :char
+          VARCHAR
+        else
+          c
+        end
+      end
+    end
+
+    PG_TYPES = {} unless defined?(PG_TYPES)
+
+    # Automatically convert the built-in numeric and text array
+    # types. to PGArray instances on retrieval if the native
+    # postgres adapter is used.
+    [ [1005, PGIntegerArray, 'int2'.freeze],
+      [1007, PGIntegerArray, 'int4'.freeze],
+      [1016, PGIntegerArray, 'int8'.freeze],
+      [1021, PGFloatArray, 'real'.freeze],
+      [1022, PGFloatArray, 'double precision'.freeze],
+      [1231, PGDecimalArray, 'numeric'.freeze],
+      [1009, PGStringArray, 'text'.freeze],
+      [1014, PGStringArray, 'char'.freeze],
+      [1015, PGStringArray, 'varchar'.freeze]
+    ].each do |ftype, klass, type|
+      meth = klass.method(:parse)
+      PG_TYPES[ftype] = lambda{|s| meth.call(s, type)}
+    end
+  end
+end
+
+class Array
+  # Return a PGArray proxy to the receiver, using a
+  # specific database type if given.  This is mostly useful
+  # as a short cut for creating PGArray objects that didn't
+  # come from the database.
+  def pg_array(type=nil)
+    Sequel::Postgres::PGArray.new(self, type)
+  end
+end

data/lib/sequel/extensions/pg_array_ops.rb

@@ -0,0 +1,220 @@
+# The pg_array_ops extension adds support to Sequel's DSL to make
+# it easier to call PostgreSQL array functions and operators. The
+# most common usage is taking an object that represents an SQL
+# identifier (such as a :symbol), and calling #pg_array on it:
+#
+#   ia = :int_array_column.pg_array
+#
+# This creates a Sequel::Postgres::ArrayOp object that can be used
+# for easier querying:
+#
+#   ia[1]     # int_array_column[1]
+#   ia[1][2]  # int_array_column[1][2]
+#
+#   ia.contains(:other_int_array_column)     # @> 
+#   ia.contained_by(:other_int_array_column) # <@
+#   ia.overlaps(:other_int_array_column)     # &&
+#   ia.concat(:other_int_array_column)       # ||
+#
+#   ia.push(1)         # int_array_column || 1
+#   ia.unshift(1)      # 1 || int_array_column
+#
+#   ia.any             # ANY(int_array_column)
+#   ia.all             # ALL(int_array_column)
+#   ia.dims            # array_dims(int_array_column)
+#   ia.length          # array_length(int_array_column, 1)
+#   ia.length(2)       # array_length(int_array_column, 2)
+#   ia.lower           # array_lower(int_array_column, 1)
+#   ia.lower(2)        # array_lower(int_array_column, 2)
+#   ia.join            # array_to_string(int_array_column, '', NULL)
+#   ia.join(':')       # array_to_string(int_array_column, ':', NULL)
+#   ia.join(':', ' ')  # array_to_string(int_array_column, ':', ' ')
+#   ia.unnest          # unnest(int_array_column)
+# 
+# See the PostgreSQL array function and operator documentation for more
+# details on what these functions and operators do.
+#
+# If you are also using the pg_array extension, you should load it before
+# loading this extension.  Doing so will allow you to use PGArray#op to get
+# an ArrayOp, allowing you to perform array operations on array literals.
+module Sequel
+  module Postgres
+    # The ArrayOp class is a simple container for a single object that
+    # defines methods that yield Sequel expression objects representing
+    # PostgreSQL array operators and functions.
+    #
+    # In the method documentation examples, assume that:
+    #
+    #   array_op = :array.pg_array
+    class ArrayOp < Sequel::SQL::Wrapper
+      CONCAT = ["(".freeze, " || ".freeze, ")".freeze].freeze
+      CONTAINS = ["(".freeze, " @> ".freeze, ")".freeze].freeze
+      CONTAINED_BY = ["(".freeze, " <@ ".freeze, ")".freeze].freeze
+      OVERLAPS = ["(".freeze, " && ".freeze, ")".freeze].freeze
+
+      # Access a member of the array, returns an SQL::Subscript instance:
+      #
+      #   array_op[1] # array[1]
+      def [](key)
+        Sequel::SQL::Subscript.new(self, [key])
+      end
+
+      # Call the ALL function:
+      #
+      #   array_op.all # ALL(array)
+      #
+      # Usually used like:
+      #
+      #   dataset.where(1=>array_op.all)
+      #   # WHERE (1 = ALL(array))
+      def all
+        function(:ALL)
+      end
+
+      # Call the ANY function:
+      #
+      #   array_op.all # ANY(array)
+      #
+      # Usually used like:
+      #
+      #   dataset.where(1=>array_op.any)
+      #   # WHERE (1 = ANY(array))
+      def any
+        function(:ANY)
+      end
+
+      # Use the contains (@>) operator:
+      #
+      #   array_op.contains(:a) # (array @> a)
+      def contains(other)
+        bool_op(CONTAINS, other)
+      end
+
+      # Use the contained by (<@) operator:
+      #
+      #   array_op.contained_by(:a) # (array <@ a)
+      def contained_by(other)
+        bool_op(CONTAINED_BY, other)
+      end
+
+      # Call the array_dims method:
+      #
+      #   array_op.dims # array_dims(array)
+      def dims
+        function(:array_dims)
+      end
+
+      # Call the array_length method:
+      #
+      #   array_op.length    # array_length(array, 1)
+      #   array_op.length(2) # array_length(array, 2)
+      def length(dimension = 1)
+        function(:array_length, dimension)
+      end
+      
+      # Call the array_lower method:
+      #
+      #   array_op.lower    # array_lower(array, 1)
+      #   array_op.lower(2) # array_lower(array, 2)
+      def lower(dimension = 1)
+        function(:array_lower, dimension)
+      end
+      
+      # Use the overlaps (&&) operator:
+      #
+      #   array_op.overlaps(:a) # (array && a)
+      def overlaps(other)
+        bool_op(OVERLAPS, other)
+      end
+
+      # Use the concatentation (||) operator:
+      #
+      #   array_op.push(:a) # (array || a)
+      #   array_op.concat(:a) # (array || a)
+      def push(other)
+        array_op(CONCAT, [self, other])
+      end
+      alias concat push
+
+      # Return the receiver.
+      def pg_array
+        self
+      end
+
+      # Call the array_to_string method:
+      #
+      #   array_op.join           # array_to_string(array, '', NULL)
+      #   array_op.to_string      # array_to_string(array, '', NULL)
+      #   array_op.join(":")      # array_to_string(array, ':', NULL)
+      #   array_op.join(":", "*") # array_to_string(array, ':', '*')
+      def to_string(joiner="", null=nil)
+        function(:array_to_string, joiner, null)
+      end
+      alias join to_string
+      
+      # Call the unnest method:
+      #
+      #   array_op.unnest # unnest(array)
+      def unnest
+        function(:unnest)
+      end
+      
+      # Use the concatentation (||) operator, reversing the order:
+      #
+      #   array_op.unshift(:a) # (a || array)
+      def unshift(other)
+        array_op(CONCAT, [other, self])
+      end
+
+      private
+
+      # Return a placeholder literal with the given str and args, wrapped
+      # in an ArrayOp, used by operators that return arrays.
+      def array_op(str, args)
+        ArrayOp.new(Sequel::SQL::PlaceholderLiteralString.new(str, args))
+      end
+
+      # Return a placeholder literal with the given str and args, wrapped
+      # in a boolean expression, used by operators that return booleans.
+      def bool_op(str, other)
+        Sequel::SQL::BooleanExpression.new(:NOOP, Sequel::SQL::PlaceholderLiteralString.new(str, [value, other]))
+      end
+
+      # Return a function with the given name, and the receiver as the first
+      # argument, with any additional arguments given.
+      def function(name, *args)
+        SQL::Function.new(name, self, *args)
+      end
+    end
+
+    module ArrayOpMethods
+      # Wrap the receiver in an ArrayOp so you can easily use the PostgreSQL
+      # array functions and operators with it.
+      def pg_array
+        ArrayOp.new(self)
+      end
+    end
+
+    if defined?(PGArray)
+      class PGArray
+        # Wrap the PGArray instance in an ArrayOp, allowing you to easily use
+        # the PostgreSQL array functions and operators with literal arrays.
+        def op
+          ArrayOp.new(self)
+        end
+      end
+    end
+  end
+
+  class SQL::GenericExpression
+    include Sequel::Postgres::ArrayOpMethods
+  end
+
+  class LiteralString
+    include Sequel::Postgres::ArrayOpMethods
+  end
+end
+
+class Symbol
+  include Sequel::Postgres::ArrayOpMethods
+end