flex_columns 1.0.0

data/lib/flex_columns/definition/field_set.rb

@@ -0,0 +1,89 @@
+require 'flex_columns/errors'
+require 'flex_columns/definition/field_definition'
+
+module FlexColumns
+  module Definition
+    # A FieldSet keeps track of a set of FieldDefinition objects for a particular flex-column contents calss. It's largely
+    # a wrapper around this set that allows you to add fields (via #field), find fields based on their name or JSON
+    # storage name, return all field names, and invoke certain delegation methods across all fields.
+    class FieldSet
+      # Create a new instance for the given class that inherits from FlexColumnContentsBase.
+      def initialize(flex_column_class)
+        @flex_column_class = flex_column_class
+        @fields = { }
+        @fields_by_json_storage_names = { }
+      end
+
+      # Defines a new field. This is passed through directly by the flex-column contents class -- its semantics are therefore
+      # exactly what the client sees. +name+ is the name of the new field, and +args+ receives any additional arguments
+      # (type, options, etc.).
+      def field(name, *args)
+        # Peel off the options
+        options = args.pop if args[-1] && args[-1].kind_of?(Hash)
+        options ||= { }
+
+        # Clean up the name
+        name = FlexColumns::Definition::FieldDefinition.normalize_name(name)
+
+        # Create a new field
+        field = FlexColumns::Definition::FieldDefinition.new(@flex_column_class, name, args, options)
+
+        # If we have a duplicate name, that's OK; we intentionally replace the existing field. But if we have a
+        # collision in the JSON storage name, and the field names are different, we want to raise an exception,
+        # because that means you actually have two _different_ fields with the same JSON storage name.
+        same_json_storage_name_field = fields_by_json_storage_names[field.json_storage_name]
+        if same_json_storage_name_field && same_json_storage_name_field.field_name != field.field_name
+          raise FlexColumns::Errors::ConflictingJsonStorageNameError.new(@flex_column_class.model_class,
+            @flex_column_class.column_name, name, same_json_storage_name_field.field_name, field.json_storage_name)
+        end
+
+        fields[name] = field
+        fields_by_json_storage_names[field.json_storage_name] = field
+      end
+
+      # Adds all delegated methods to both the +column_dynamic_methods_module+, which should be included into the
+      # flex-column contents class, and the +model_dynamic_methods_module+, which should be included into the
+      # +model_class+. The +model_class+ itself is also passed here; this is used in the FieldDefinition just to make
+      # sure we don't define methods that collide with column names or other method names on the model class itself.
+      def add_delegated_methods!(column_dynamic_methods_module, model_dynamic_methods_module, model_class)
+        each_field do |field_definition|
+          field_definition.add_methods_to_flex_column_class!(column_dynamic_methods_module)
+          field_definition.add_methods_to_model_class!(model_dynamic_methods_module, model_class)
+        end
+      end
+
+      # Returns the names of all defined fields, in no particular order.
+      def all_field_names
+        fields.keys
+      end
+
+      # Adds delegated methods, as appropriate for IncludeFlexColumns#include_flex_columns_from, to the given
+      # DynamicMethodsModule. +association_name+ is the name of the method on the target class that, when called, will
+      # return the associated model object of the class on which this flex column is defined (_i.e._, the association
+      # name); +target_class+ is the class into which the DynamicMethodsModule is included, so we can check to make sure
+      # we're not clobbering methods that we really shouldn't clobber, and +options+ is any options passed along.
+      def include_fields_into(dynamic_methods_module, association_name, target_class, options)
+        each_field do |field_definition|
+          field_definition.add_methods_to_included_class!(dynamic_methods_module, association_name, target_class, options)
+        end
+      end
+
+      # Returns the field with the given name, or +nil+ if there is no such field
+      def field_named(field_name)
+        fields[FlexColumns::Definition::FieldDefinition.normalize_name(field_name)]
+      end
+
+      # Returns the field with the given JSON storage name, or +nil+ if there is no such field.
+      def field_with_json_storage_name(json_storage_name)
+        fields_by_json_storage_names[FlexColumns::Definition::FieldDefinition.normalize_name(json_storage_name)]
+      end
+
+      private
+      attr_reader :fields, :fields_by_json_storage_names
+
+      def each_field(&block)
+        fields.each { |name, field| block.call(field) }
+      end
+    end
+  end
+end

data/lib/flex_columns/definition/flex_column_contents_class.rb

@@ -0,0 +1,327 @@
+module FlexColumns
+  module Definition
+    # When you declare a flex column, we actually generate a brand-new Class for that column; instances of that flex
+    # column are instances of this new Class. This class acquires functionality from two places: FlexColumnContentsBase,
+    # which defines its instance methods, and FlexColumnContentsClass, which defines its class methods. (While
+    # FlexColumnContentsBase is an actual Class, FlexColumnContentsClass is a Module that FlexColumnContentsBase
+    # +extend+s. Both could be combined, but, simply for readability and maintainability, it was better to make them
+    # separate.)
+    #
+    # This Module therefore defines the methods that are available on a flex-column class -- directly from inside
+    # the block passed to +flex_column+, for example.
+    module FlexColumnContentsClass
+      # By default, how long does the generated JSON have to be before we'll try compressing it?
+      DEFAULT_MAX_JSON_LENGTH_BEFORE_COMPRESSION = 200
+
+      # Given a string from storage in +storage_string+, and an object that responds to ColumnData's +data_source+
+      # protocol for describing where data came from, create the appropriate ColumnData object to represent that data.
+      # (+storage_string+ can absolutely be +nil+, in case there is no data yet.)
+      #
+      # This is used by instances of the generated Class to create the ColumnData object that does most of the work of
+      # actually serializing/deserializing JSON and storing data for that instance.
+      def _flex_columns_create_column_data(storage_string, data_source)
+        ensure_setup!
+
+        storage = case column.type
+        when :binary, :text, :json then column.type
+        when :string then :text
+        else raise "Unknown storage type: #{column.type.inspect}"
+        end
+
+        create_options = {
+          :storage_string => storage_string,
+          :data_source    => data_source,
+          :unknown_fields => options[:unknown_fields] || :preserve,
+          :length_limit   => column.limit,
+          :storage        => storage,
+          :binary_header  => true,
+          :null           => column.null
+        }
+
+        create_options[:binary_header] = false if options.has_key?(:header) && (! options[:header])
+
+        if (! options.has_key?(:compress))
+          create_options[:compress_if_over_length] = DEFAULT_MAX_JSON_LENGTH_BEFORE_COMPRESSION
+        elsif options[:compress]
+          create_options[:compress_if_over_length] = options[:compress]
+        end
+
+        FlexColumns::Contents::ColumnData.new(field_set, create_options)
+      end
+
+      # This is what gets called when you declare a field inside a flex column.
+      def field(name, *args)
+        ensure_setup!
+        field_set.field(name, *args)
+      end
+
+      # Returns the field with the given name, or nil if there is no such field.
+      def field_named(name)
+        ensure_setup!
+        field_set.field_named(name)
+      end
+
+      # Returns the field that stores its JSON under the given key (+json_storage_name+), or nil if there is no such
+      # field.
+      def field_with_json_storage_name(json_storage_name)
+        ensure_setup!
+        field_set.field_with_json_storage_name(json_storage_name)
+      end
+
+      # Is this a flex-column class? Of course it is, by definition. We just use this for argument validation in some
+      # places.
+      def is_flex_column_class?
+        true
+      end
+
+      # Tells this flex column that you want to include its methods into the given +dynamic_methods_module+, which is
+      # included in the given +target_class+. (We only use +target_class+ to make sure we don't define methods that
+      # are already present on the given +target_class+.) +association_name+ is the name of the association that,
+      # from the given +target_class+, will return a model instance that contains this flex column.
+      #
+      # +options+ specifies options for the inclusion; it can specify +:visibility+ to change whether methods are
+      # public or private, +:delegate+ to turn off delegation of anything other than the flex column itself, or
+      # +:prefix+ to set a prefix for the delegated method names.
+      def include_fields_into(dynamic_methods_module, association_name, target_class, options)
+        ensure_setup!
+
+        cn = column_name
+        mn = column_name.to_s
+        mn = "#{options[:prefix]}_#{mn}" if options[:prefix]
+
+        # Make sure we don't overwrite some #method_missing magic that defines a column accessor, or something
+        # similar.
+        if target_class._flex_columns_safe_to_define_method?(mn)
+          dynamic_methods_module.define_method(mn) do
+            associated_object = send(association_name) || send("build_#{association_name}")
+            associated_object.send(cn)
+          end
+          dynamic_methods_module.private(mn) if options[:visibility] == :private
+        end
+
+        unless options.has_key?(:delegate) && (! options[:delegate])
+          add_custom_methods!(dynamic_methods_module, target_class, options)
+          field_set.include_fields_into(dynamic_methods_module, association_name, target_class, options)
+        end
+      end
+
+      # Given an instance of the model that this flex column is defined on, return the appropriate flex-column
+      # object for that instance. This simply delegates to #_flex_column_object_for on that model instance.
+      def object_for(model_instance)
+        ensure_setup!
+        model_instance._flex_column_object_for(column.name)
+      end
+
+      # When we delegate methods, what should we prefix them with (if anything)?
+      def delegation_prefix
+        ensure_setup!
+        options[:prefix].try(:to_s)
+      end
+
+      # When we delegate methods, should we delegate them at all (returns +nil+), publicly (+:public+), or
+      # privately (+:private+)?
+      def delegation_type
+        ensure_setup!
+        return :public if (! options.has_key?(:delegate))
+
+        case options[:delegate]
+        when nil, false then nil
+        when true, :public then :public
+        when :private then :private
+        # OK to raise an untyped error here -- we should've caught this in #validate_options.
+        else raise "Impossible value for :delegate: #{options[:delegate]}"
+        end
+      end
+
+      # What's the name of the actual model column this flex-column uses? Returns a Symbol.
+      def column_name
+        ensure_setup!
+        column.name.to_sym
+      end
+
+      # What are the names of all fields defined on this flex column?
+      def all_field_names
+        field_set.all_field_names
+      end
+
+      # Given a model instance, do we need to save this column? This is true under one of two cases:
+      #
+      # * Someone has changed ("touched") at least one of the flex-column fields (or called #touch! on it);
+      # * The column is non-NULL, and there's no data in it right now. (Saving it will populate it with an empty string.)
+      def requires_serialization_on_save?(model)
+        maybe_flex_object = model._flex_column_object_for(column_name, false)
+        out = true if maybe_flex_object && maybe_flex_object.touched?
+        out ||= true if ((! column.null) && (! model[column_name]))
+        out
+      end
+
+      # Are fields in this flex column private by default?
+      def fields_are_private_by_default?
+        ensure_setup!
+        options[:visibility] == :private
+      end
+
+      # This is, for all intents and purposes, the initializer (constructor) for this module. But because it's a module
+      # (and has to be), this can't actually be #initialize. (Another way of saying it: objects have initializers;
+      # classes do not.)
+      #
+      # You must call this method exactly once for each class that extends this module, and before you call any other
+      # method.
+      #
+      # +model_class+ must be the ActiveRecord model class for this flex column. +column_name+ must be the name of
+      # the column that you're using as a flex column. +options+ can contain any of:
+      #
+      # [:visibility] If +:private+, then all field accessors (readers and writers) will be private by default, unless
+      #               overridden in their field declaration.
+      # [:delegate] If specified and +false+ or +nil+, then field accessors and custom methods defined in this class
+      #             will not be automatically delegated to from the +model_class+.
+      # [:prefix] If specified (as a Symbol or String), then field accessors and custom methods delegated from the
+      #           +model_class+ will be prefixed with this string, followed by an underscore.
+      # [:unknown_fields] If specified and +:delete+, then, if the JSON string for an instance contains fields that
+      #                   aren't declared in this class, they will be removed from the JSON when saving back out to
+      #                   the database. This is dangerous, but powerful, if you want to keep your data clean.
+      # [:compress] If specified and +false+, this column will never be compressed. If specified as a number, then,
+      #             when serializing data, we'll try to compress it if the uncompressed version is at least that many
+      #             bytes long; we'll store the compressed version if it's no more than 95% as long as the uncompressed
+      #             version. The default is 200. Also note that compression requires a binary storage type for the
+      #             underlying column.
+      # [:header] If the underlying column is of binary storage type, then, by default, we use a tiny header to indicate
+      #           what kind of data is stored there and whether it's compressed or not. If this is set to +false+,
+      #           disables this header (and therefore also disables compression).
+      def setup!(model_class, column_name, options = { }, &block)
+        raise ArgumentError, "You can't call setup! twice!" if @model_class || @column
+
+        # Make really sure we're being declared in the right kind of class.
+        unless model_class.kind_of?(Class) && model_class.respond_to?(:has_any_flex_columns?) && model_class.has_any_flex_columns?
+          raise ArgumentError, "Invalid model class: #{model_class.inspect}"
+        end
+
+        raise ArgumentError, "Invalid column name: #{column_name.inspect}" unless column_name.kind_of?(Symbol)
+
+        column = model_class.columns.detect { |c| c.name.to_s == column_name.to_s }
+        unless column
+          raise FlexColumns::Errors::NoSuchColumnError, %{You're trying to define a flex column #{column_name.inspect}, but
+  the model you're defining it on, #{model_class.name}, seems to have no column
+  named that.
+
+  It has columns named: #{model_class.columns.map(&:name).sort_by(&:to_s).join(", ")}.}
+        end
+
+        unless column.type == :binary || column.text? || column.sql_type == "json" # for PostgreSQL >= 9.2, which has a native JSON data type
+          raise FlexColumns::Errors::InvalidColumnTypeError, %{You're trying to define a flex column #{column_name.inspect}, but
+  that column (on model #{model_class.name}) isn't of a type that accepts text.
+  That column is of type: #{column.type.inspect}.}
+        end
+
+        validate_options(options)
+
+        @model_class = model_class
+        @column = column
+        @options = options
+        @field_set = FlexColumns::Definition::FieldSet.new(self)
+
+        class_name = "#{column_name.to_s.camelize}FlexContents".to_sym
+        @model_class.send(:remove_const, class_name) if @model_class.const_defined?(class_name)
+        @model_class.const_set(class_name, self)
+
+        # Keep track of which methods were present before and after calling the block that was passed in; this is how
+        # we know which methods were declared custom, so we know which ones to add delegation for.
+        methods_before = instance_methods
+        block_result = class_eval(&block) if block
+        @custom_methods = (instance_methods - methods_before).map(&:to_sym)
+        block_result
+      end
+
+      # Tells this class to re-publish all its methods to the DynamicMethodsModule it uses internally, and to the
+      # model class it's a part of.
+      #
+      # Because Rails in development mode is constantly redefining classes, and we don't want old cruft that you've
+      # removed to hang around, we use a "remove absolutely all methods, then add back only what's defined now"
+      # strategy.
+      def sync_methods!
+        @dynamic_methods_module ||= FlexColumns::Util::DynamicMethodsModule.new(self, :FlexFieldsDynamicMethods)
+        @dynamic_methods_module.remove_all_methods!
+
+        field_set.add_delegated_methods!(@dynamic_methods_module, model_class._flex_column_dynamic_methods_module, model_class)
+
+        if delegation_type
+          add_custom_methods!(model_class._flex_column_dynamic_methods_module, model_class,
+            :visibility => (delegation_type == :private ? :private : :public))
+        end
+      end
+
+      attr_reader :model_class, :column
+
+      private
+      attr_reader :fields, :options, :custom_methods, :field_set
+
+      # Takes all custom methods defined on this flex-column class, and adds delegates to them to the given
+      # +dynamic_methods_module+. +target_class+ is checked before each one to make sure we don't have a conflict.
+      def add_custom_methods!(dynamic_methods_module, target_class, options = { })
+        cn = column_name
+
+        custom_methods.each do |custom_method|
+          mn = custom_method.to_s
+          mn = "#{options[:prefix]}_#{mn}" if options[:prefix]
+
+          if target_class._flex_columns_safe_to_define_method?(mn)
+            dynamic_methods_module.define_method(mn) do |*args, &block|
+              flex_object = _flex_column_object_for(cn)
+              flex_object.send(custom_method, *args, &block)
+            end
+
+            dynamic_methods_module.private(custom_method) if options[:visibility] == :private
+          end
+        end
+      end
+
+      # Check all of our options to make sure they're correct. This is pretty defensive programming, but it is SO
+      # much nicer to get an error on startup if you've specified anything incorrectly than way on down the line,
+      # possibly in production, when it really matters.
+      def validate_options(options)
+        unless options.kind_of?(Hash)
+          raise ArgumentError, "You must pass a Hash, not: #{options.inspect}"
+        end
+
+        options.assert_valid_keys(:visibility, :prefix, :delegate, :unknown_fields, :compress, :header)
+
+        unless [ nil, :private, :public ].include?(options[:visibility])
+          raise ArgumentError, "Invalid value for :visibility: #{options[:visibility.inspect]}"
+        end
+
+        unless [ :delete, :preserve, nil ].include?(options[:unknown_fields])
+          raise ArgumentError, "Invalid value for :unknown_fields: #{options[:unknown_fields].inspect}"
+        end
+
+        unless [ true, false, nil ].include?(options[:compress]) || options[:compress].kind_of?(Integer)
+          raise ArgumentError, "Invalid value for :compress: #{options[:compress].inspect}"
+        end
+
+        unless [ true, false, nil ].include?(options[:header])
+          raise ArgumentError, "Invalid value for :header: #{options[:header].inspect}"
+        end
+
+        case options[:prefix]
+        when nil then nil
+        when String, Symbol then nil
+        else raise ArgumentError, "Invalid value for :prefix: #{options[:prefix].inspect}"
+        end
+
+        unless [ nil, true, false, :private, :public ].include?(options[:delegate])
+          raise ArgumentError, "Invalid value for :delegate: #{options[:delegate].inspect}"
+        end
+
+        if options[:visibility] == :private && options[:delegate] == :public
+          raise ArgumentError, "You can't have public delegation if methods in the flex column are private; this makes no sense, as methods in the model class would have *greater* visibility than methods on the flex column itself"
+        end
+      end
+
+      # Make sure someone has called setup! previously.
+      def ensure_setup!
+        unless @model_class
+          raise "You must call #setup! on this class before calling this method."
+        end
+      end
+    end
+  end
+end

data/lib/flex_columns/errors.rb

@@ -0,0 +1,236 @@
+require 'flex_columns/util/string_utils'
+
+module FlexColumns
+  # This module contains definitions for all errors thrown by +flex_columns+. One of the goals of +flex_columns+ is to,
+  # when an error occurs, raise an exception that has a great amount of detail about what happened -- in general, it
+  # should be enough to know exactly where any invalid or problematic data came from, such as the row in the database
+  # containing bad data, invalidly-encoded characters, or similar.
+  module Errors
+    # FlexColumns::Errors::Base: all +flex_columns+ errors inherit from this class.
+    class Base < StandardError; end
+
+
+    # FlexColumns::Errors::FieldError: all errors having to do with field definition inherit from this class.
+    class FieldError < Base; end
+
+    # Raised when you try to read or write data for a field that isn't defined.
+    class NoSuchFieldError < FieldError
+      attr_reader :data_source, :field_name, :all_field_names
+
+      def initialize(data_source, field_name, all_field_names)
+        @data_source = data_source
+        @field_name = field_name
+        @all_field_names = all_field_names
+
+        super(%{You tried to set field #{field_name.inspect} of #{data_source.describe_flex_column_data_source}.
+However, there is no such field defined on that flex column; the defined fields are:
+
+  #{all_field_names.join(", ")}})
+      end
+    end
+
+    # Raised when you try to define a field with the same JSON storage name, but a different field name, as a
+    # previously-defined field.
+    class ConflictingJsonStorageNameError < FieldError
+      attr_reader :model_class, :column_name, :new_field_name, :existing_field_name, :json_storage_name
+
+      def initialize(model_class, column_name, new_field_name, existing_field_name, json_storage_name)
+        @model_class = model_class
+        @column_name = column_name
+        @new_field_name = new_field_name
+        @existing_field_name = existing_field_name
+        @json_storage_name = json_storage_name
+
+        super(%{On class #{model_class.name}, flex column #{column_name.inspect}, you're trying to define a field,
+#{new_field_name.inspect}, that has a JSON storage name of #{json_storage_name.inspect},
+but there's already another field, #{existing_field_name.inspect}, that uses that same JSON storage name.
+
+These fields would conflict in the JSON store, and thus this is not allowed.})
+      end
+    end
+
+
+    # FlexColumns::Errors::DefinitionError: all errors having to do with definition of a flex column itself (not fields,
+    # but the whole column) inherit from this class.
+    class DefinitionError < Base; end
+
+    # Raised when you try to define a flex_column for a column that doesn't exist in the database (at least according
+    # to the model class).
+    class NoSuchColumnError < DefinitionError; end
+
+    # Raised when you try to define a flex_column for a column that isn't of a valid type for that -- for example, an
+    # integer or a boolean column.
+    class InvalidColumnTypeError < DefinitionError; end
+
+
+    # FlexColumns::Errors::DataError: all errors having to do with the data present in a flex column in the database
+    # inherit from this class.
+    class DataError < Base; end
+
+    # Raised when you try to store enough data in a flex column that the generated JSON is too long to fit into the
+    # column.
+    class JsonTooLongError < DataError
+      attr_reader :data_source, :limit, :json_string
+
+      def initialize(data_source, limit, json_string)
+        @data_source = data_source
+        @limit = limit
+        @json_string = json_string
+
+        super(%{When trying to serialize JSON for #{data_source.describe_flex_column_data_source},
+the JSON produced was too long to fit in the database.
+We produced #{json_string.length} characters of JSON, but the
+database's limit for that column is #{limit} characters.
+
+The JSON we produced was:
+
+  #{FlexColumns::Util::StringUtils.abbreviated_string(json_string)}})
+      end
+    end
+
+    # FlexColumns::Errors::InvalidDataInDatabaseError: all errors raised because something is wrong with the data
+    # already stored in the database for a particular row and column.
+    class InvalidDataInDatabaseError < DataError
+      attr_reader :data_source, :raw_string, :additional_message
+
+      def initialize(data_source, raw_string, additional_message = nil)
+        @data_source = data_source
+        @raw_string = raw_string
+        @additional_message = additional_message
+
+        super(create_message)
+      end
+
+      private
+      def create_message
+        out = %{When parsing the JSON in #{data_source.describe_flex_column_data_source}, which is:
+
+#{FlexColumns::Util::StringUtils.abbreviated_string(raw_string)}
+
+}
+        out += additional_message if additional_message
+        out
+      end
+    end
+
+    # Raised when the data in the database appears to be GZip'ed, but we can't decompress that data.
+    class InvalidCompressedDataInDatabaseError < InvalidDataInDatabaseError
+      attr_reader :source_exception
+
+      def initialize(data_source, raw_string, source_exception)
+        @source_exception = source_exception
+        super(data_source, raw_string)
+      end
+
+      private
+      def create_message
+        super + %{, we got an exception when trying to decompress the data:
+
+#{source_exception} (#{source_exception.class.name})}
+      end
+    end
+
+    # Raised when the data in the database appears to have a +flex_columns+ header on it, but the version number is
+    # not something we support.
+    class InvalidFlexColumnsVersionNumberInDatabaseError < InvalidDataInDatabaseError
+      attr_reader :version_number_in_database, :max_version_number_supported
+
+      def initialize(data_source, raw_string, version_number_in_database, max_version_number_supported)
+        @version_number_in_database = version_number_in_database
+        @max_version_number_supported = max_version_number_supported
+        super(data_source, raw_string)
+      end
+
+      private
+      def create_message
+        super + %{, we got a version number in the database, #{version_number_in_database}, which is greater than our maximum supported version number, #{max_version_number_supported}.}
+      end
+    end
+
+    # Raised when the data in the database is not parseable as JSON (via JSON.parse). Note that we take special care to
+    # exclude characters from the message that aren't in a valid encoding, as this is one of the major causes of JSON
+    # parsing failures in some situations...and we really don't want to create an exception that itself has a message
+    # with encoding problems.
+    class UnparseableJsonInDatabaseError < InvalidDataInDatabaseError
+      attr_reader :source_exception
+
+      def initialize(data_source, raw_string, source_exception)
+        @source_exception = source_exception
+        super(data_source, raw_string)
+      end
+
+      private
+      def create_message
+        source_message = source_exception.message
+
+        if source_message.respond_to?(:force_encoding)
+          source_message.force_encoding("UTF-8")
+          source_message = source_message.chars.select { |c| c.valid_encoding? }.join
+        end
+
+        super + %{, we got an exception: #{source_message} (#{source_exception.class.name})}
+      end
+    end
+
+    # Raised when the string stored in the database is not correctly encoded. We check for this situation before we
+    # even try to parse the string as JSON, because the kind of errors you get from this problem are otherwise
+    # maddeningly difficult to deal with -- partly because the exceptions themselves often end up with bad encoding.
+    #
+    # This class does a lot of work to filter out invalid characters, show them just as hex, and show you where into the
+    # string they occur. This is, again, so we don't make things worse by raising an exception with invalid characters
+    # in its message, and so that you can figure out where the problems are.
+    class IncorrectlyEncodedStringInDatabaseError < InvalidDataInDatabaseError
+      attr_reader :invalid_chars_as_array, :raw_data_as_array, :first_bad_position
+
+      def initialize(data_source, raw_string)
+        @raw_data_as_array = raw_string.chars.to_a
+        @valid_chars_as_array = [ ]
+        @invalid_chars_as_array = [ ]
+        @raw_data_as_array.each_with_index do |c, i|
+          if (! c.valid_encoding?)
+            @invalid_chars_as_array << c
+            @first_bad_position ||= i
+          else
+            @valid_chars_as_array << c
+          end
+        end
+        @first_bad_position ||= :unknown
+
+        super(data_source, @valid_chars_as_array.join)
+      end
+
+      private
+      def create_message
+        extra = %{\n\nThere are #{invalid_chars_as_array.length} invalid characters out of #{raw_data_as_array.length} total characters.
+(The string above showing the original JSON omits them, so that it's actually a valid String.)
+The first bad character occurs at position #{first_bad_position}.
+
+Some of the invalid chars are (in hex):
+
+    }
+
+        extra += invalid_chars_as_array[0..19].map { |c| c.unpack("H*") }.join(" ")
+
+        super + extra
+      end
+    end
+
+    # Raised when the JSON in the database is invalid -- not because it's not actually JSON, but because it doesn't
+    # represent a Hash.
+    class InvalidJsonInDatabaseError < InvalidDataInDatabaseError
+      attr_reader :returned_data
+
+      def initialize(data_source, raw_string, returned_data)
+        @returned_data = returned_data
+        super(data_source, raw_string)
+      end
+
+      private
+      def create_message
+        super + %{, the JSON returned wasn't a Hash, but rather #{returned_data.class.name}:
+
+#{FlexColumns::Util::StringUtils.abbreviated_string(returned_data.inspect)}}
+      end
+    end
+  end
+end