easy_ml 0.1.4 → 0.2.0.pre.rc1

data/app/models/easy_ml/column_history.rb

@@ -0,0 +1,30 @@
+# == Schema Information
+#
+# Table name: easy_ml_column_histories
+#
+#  id                  :bigint           not null, primary key
+#  column_id           :integer          not null
+#  dataset_id          :integer          not null
+#  name                :string           not null
+#  description         :string
+#  datatype            :string
+#  polars_datatype     :string
+#  is_target           :boolean
+#  hidden              :boolean          default(FALSE)
+#  drop_if_null        :boolean          default(FALSE)
+#  preprocessing_steps :json
+#  sample_values       :json
+#  statistics          :json
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  history_started_at  :datetime         not null
+#  history_ended_at    :datetime
+#  history_user_id     :integer
+#  snapshot_id         :string
+#
+module EasyML
+  class ColumnHistory < ActiveRecord::Base
+    self.table_name = "easy_ml_column_histories"
+    include Historiographer::History
+  end
+end

data/app/models/easy_ml/column_list.rb

@@ -0,0 +1,122 @@
+module EasyML
+  module ColumnList
+    def sync
+      return unless dataset.schema.present?
+
+      EasyML::Column.transaction do
+        col_names = syncable
+        existing_columns = where(name: col_names)
+        import_new(col_names, existing_columns)
+        update_existing(existing_columns)
+        delete_missing(existing_columns)
+      end
+    end
+
+    def one_hots
+      column_list.select(&:one_hot?)
+    end
+
+    def allowed_categories
+      one_hots.reduce({}) do |h, col|
+        h.tap do
+          h[col.name] = col.allowed_categories
+        end
+      end
+    end
+
+    def one_hot?(column)
+      one_hots.map(&:name).detect do |one_hot_col|
+        column.start_with?(one_hot_col)
+      end
+    end
+
+    def virtual_column?(column)
+      false
+    end
+
+    def syncable
+      dataset.processed_schema.keys.select do |col|
+        !one_hot?(col) &&
+          !virtual_column?(col)
+      end
+    end
+
+    def column_list
+      self
+    end
+
+    def dataset
+      proxy_association.owner
+    end
+
+    private
+
+    def import_new(new_columns, existing_columns)
+      new_columns = new_columns - existing_columns.map(&:name)
+      cols_to_insert = new_columns.map do |col_name|
+        EasyML::Column.new(
+          name: col_name,
+          dataset_id: dataset.id,
+        )
+      end
+      EasyML::Column.import(cols_to_insert)
+    end
+
+    def update_existing(existing_columns)
+      stats = dataset.statistics
+      use_processed = dataset.processed.data(limit: 1).present?
+      cached_sample = use_processed ? dataset.processed.data(limit: 10, all_columns: true) : dataset.raw.data(limit: 10, all_columns: true)
+      existing_types = existing_columns.map(&:name).zip(existing_columns.map(&:datatype)).to_h
+      polars_types = cached_sample.columns.zip((cached_sample.dtypes.map do |dtype|
+        EasyML::Data::PolarsColumn.polars_to_sym(dtype).to_s
+      end)).to_h
+
+      existing_columns.each do |column|
+        new_polars_type = polars_types[column.name]
+        existing_type = existing_types[column.name]
+        schema_type = dataset.schema[column.name]
+
+        # Keep both datatype and polars_datatype if it's an ordinal encoding case
+        if column.ordinal_encoding?
+          actual_type = existing_type
+          actual_schema_type = existing_type
+        else
+          actual_type = new_polars_type
+          actual_schema_type = schema_type
+        end
+
+        if column.one_hot?
+          base = dataset.raw
+          processed = stats.dig("raw", column.name).dup
+          processed["null_count"] = 0
+          actual_schema_type = "categorical"
+          actual_type = "categorical"
+        else
+          base = use_processed ? dataset.processed : dataset.raw
+          processed = stats.dig("processed", column.name)
+        end
+        sample_values = base.send(:data, unique: true, limit: 5, all_columns: true, select: column.name)[column.name].to_a.uniq[0...5]
+
+        column.assign_attributes(
+          statistics: {
+            raw: stats.dig("raw", column.name),
+            processed: processed,
+          },
+          datatype: actual_schema_type,
+          polars_datatype: actual_type,
+          sample_values: sample_values,
+        )
+      end
+      EasyML::Column.import(existing_columns.to_a,
+                            { on_duplicate_key_update: { columns: %i[statistics datatype polars_datatype
+                                                                   sample_values] } })
+    end
+
+    def delete_missing(existing_columns)
+      raw_cols = dataset.raw.train(all_columns: true, limit: 1).columns
+      raw_cols = where(name: raw_cols)
+      columns_to_delete = column_list - existing_columns - raw_cols
+      columns_to_delete.each(&:destroy!)
+    end
+  end
+end

data/app/models/easy_ml/concerns/configurable.rb

@@ -0,0 +1,61 @@
+module EasyML
+  module Concerns
+    module Configurable
+      extend ActiveSupport::Concern
+
+      included do
+        is_model = ancestors.include?(ActiveRecord::Base)
+        self.filter_attributes += [:configuration] if is_model
+        class_attribute :configuration_attributes, instance_writer: false, default: []
+
+        after_initialize :read_from_configuration if is_model
+        before_save :store_in_configuration if is_model
+      end
+
+      class_methods do
+        def add_configuration_attributes(*attrs)
+          self.configuration_attributes += attrs
+          self.configuration_attributes = self.configuration_attributes.uniq
+
+          # Define attr_accessor for each configuration attribute
+          attrs.each do |attr|
+            attr_accessor attr unless method_defined?(attr)
+          end
+        end
+      end
+
+      private
+
+      def read_from_configuration
+        return unless configuration
+
+        self.class.configuration_attributes.each do |attr|
+          next unless configuration.key?(attr.to_s)
+
+          instance_variable_set("@#{attr}", configuration[attr.to_s])
+        end
+      end
+
+      def store_in_configuration
+        serialized = self.class.configuration_attributes.each_with_object({}) do |attr, hash|
+          value = public_send(attr)
+          hash[attr] = serialize_value(value)
+        end
+
+        self.configuration = (configuration || {}).merge(serialized)
+      end
+
+      def serialize_value(value)
+        return value if basic_type?(value)
+        return value.to_h if value.respond_to?(:to_h)
+        return value.as_json if value.respond_to?(:as_json)
+
+        value.to_s
+      end
+
+      def basic_type?(value)
+        [String, Symbol, Integer, Float, TrueClass, FalseClass, NilClass, Hash, Array].any? { |type| value.is_a?(type) }
+      end
+    end
+  end
+end

data/app/models/easy_ml/concerns/versionable.rb

@@ -0,0 +1,19 @@
+module EasyML
+  module Concerns
+    module Versionable
+      extend ActiveSupport::Concern
+
+      included do
+        def bump_version(force: false)
+          return version if version.present? && !force
+
+          prev_version = version
+          timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
+          timestamp = (prev_version.to_i + 1).to_s if timestamp.to_i <= prev_version.to_i
+
+          self.version = timestamp
+        end
+      end
+    end
+  end
+end