easy_ml 0.2.0.pre.rc56 → 0.2.0.pre.rc58

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2eb7d933162cc05c64a1ea7c21c65f9c80283a1ae45f37226282c503607008f
-  data.tar.gz: e25a1b7b1970753ae6f65917943607102ad52ef9f90831de0660563441448975
+  metadata.gz: 1a25c50b89c079e7e62f52d1f5a52ef16f3d7bc9b388fcee9a7b0983148de9cd
+  data.tar.gz: bfcf0d06fbe498ccc70251c144649d7fd6699c2bdc4a9acbf5866e60ce04c7bd
 SHA512:
-  metadata.gz: 6fc39e6b2838ab6242df1848411450764b0434b63bb7e4f1cb60151850e794f1d0a71a956b70b7ca78e159003efc6947d7dbaad9010c7c6899a1baeb8c7570b2
-  data.tar.gz: f50ceecc6935fea0c1f82e5b76beaef2e6ee329087d0f7b7739d4a88b4738d9ed37d13acd47bf12d118092c0b66774772dab9a30f60e12c3854360329caacfa9
+  metadata.gz: 77186b1d2d7558db7d128e03c68f8632af6f28be4b1bf2daa71ac804abbcc5a26470fe29a802258d326048155fea46a7f707e46fc88cf8571af5f30cb870d839
+  data.tar.gz: a22ba3e21ab32e64674033f0c83d023e41c3c5f158117be5fa8b85f1865a4bfc39bda72042e4c1ff80277d4d664fcc694349f7a377933c7c52f23417c299618b

data/app/controllers/easy_ml/apis_controller.rb

@@ -0,0 +1,8 @@
+module EasyML
+  class APIsController < ApplicationController
+    def show
+      model = EasyML::Model.find_by!(slug: params[:model])
+      render json: { data: model.api_fields }
+    end
+  end
+end

data/app/controllers/easy_ml/models_controller.rb

@@ -53,6 +53,9 @@ module EasyML
         flash[:notice] = "Model was successfully created."
         redirect_to easy_ml_models_path
       else
+        errors = model.errors.to_hash(true)
+        values = errors.values.flatten
+        flash.now[:error] = values.join(", ")
         render inertia: "pages/NewModelPage", props: {
           datasets: EasyML::Dataset.all.map do |dataset|
             dataset.slice(:id, :name, :num_rows)

data/app/controllers/easy_ml/predictions_controller.rb

@@ -3,6 +3,11 @@ module EasyML
     skip_before_action :verify_authenticity_token, only: [:create]
 
     def create
+      slug = params[:model]
+      unless EasyML::Model.find_by(slug: slug).inference_version.present?
+        return render json: { error: "Model not found" }, status: :not_found
+      end
+
       unless params.key?(:input)
         return render json: { error: "Must provide key: input" }, status: :not_found
       end
@@ -12,17 +17,17 @@ module EasyML
         return render json: { error: "Input must be a hash" }, status: :not_found
       end
 
-      model_name = params[:model]
-      unless EasyML::Model.find_by(name: model_name).present?
-        return render json: { error: "Model not found" }, status: :not_found
+      valid, fields = EasyML::Predict.validate_input(slug, input)
+      unless valid
+        return render json: { error: "Missing required fields: #{fields}" }, status: :not_found
       end
 
-      prediction = EasyML::Predict.predict(model_name, input)
+      prediction = EasyML::Predict.predict(slug, input)
 
       render json: { prediction: EasyML::PredictionSerializer.new(prediction).serializable_hash.dig(:data, :attributes) }, status: :ok
     rescue ActiveRecord::RecordNotFound
       render json: { error: "Model not found" }, status: :not_found
-    rescue StandardError => e
+    rescue => e
       render json: { error: e.message }, status: :unprocessable_entity
     end
   end

data/app/frontend/components/ModelForm.tsx

@@ -74,7 +74,7 @@ export function ModelForm({ initialData, datasets, constants, isEditing, errors:
       dataset_id: initialData?.dataset_id || '',
       task: initialData?.task || 'classification',
       objective: initialData?.objective || 'binary:logistic',
-      metrics: initialData?.metrics || ['accuracy'],
+      metrics: initialData?.metrics || ['accuracy_score'],
       retraining_job_attributes: initialData?.retraining_job ? {
         id: initialData.retraining_job.id,
         frequency: initialData.retraining_job.frequency,

data/app/frontend/components/SearchableSelect.tsx

@@ -61,7 +61,6 @@ export const SearchableSelect = forwardRef<HTMLButtonElement, SearchableSelectPr
     }, [isOpen]);
 
     const handleOptionClick = (optionValue: Option['value'], e: React.MouseEvent) => {
-      debugger;
       e.preventDefault();
       e.stopPropagation();
       onChange(optionValue);

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -250,7 +250,7 @@ export function PreprocessingConfig({
     setIsEditingDescription(true);
   };
 
-  let nullCount = (column.statistics?.processed.null_count || column.statistics?.raw.null_count) || 0;
+  let nullCount = (column.statistics?.processed.null_count || column.statistics?.raw?.null_count) || 0;
   const nullPercentage = nullCount && column.statistics?.raw.num_rows
     ? ((nullCount / column.statistics.raw.num_rows) * 100)
     : 0;

data/app/frontend/pages/DatasourcesPage.tsx

@@ -49,12 +49,10 @@ export default function DatasourcesPage({ datasources }: { datasources: Datasour
         preserveScroll: true, // Keeps the scroll position
         preserveState: true,  // Keeps the form state
         onSuccess: (e) => {
-          debugger;
           console.log("SUCCESS")
           // The page will automatically refresh with new data
         },
         onError: () => {
-          debugger;
           // Handle error case if needed
           console.error('Failed to sync datasource');
         }

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -5,6 +5,7 @@ module EasyML
     @queue = :easy_ml
 
     def self.perform(batch_id, options = {})
+      puts "Performing compute feature job with options #{options}"
       begin
         options.symbolize_keys!
         feature_id = options.dig(:feature_id)

data/app/models/easy_ml/column.rb

@@ -8,7 +8,7 @@
 #  description         :string
 #  datatype            :string
 #  polars_datatype     :string
-#  is_target           :boolean
+#  is_target           :boolean          default(FALSE)
 #  hidden              :boolean          default(FALSE)
 #  drop_if_null        :boolean          default(FALSE)
 #  preprocessing_steps :json
@@ -17,6 +17,8 @@
 #  created_at          :datetime         not null
 #  updated_at          :datetime         not null
 #  is_date_column      :boolean          default(FALSE)
+#  computed_by         :string
+#  is_computed         :boolean          default(FALSE)
 #
 module EasyML
   class Column < ActiveRecord::Base
@@ -30,7 +32,6 @@ module EasyML
     validates :name, uniqueness: { scope: :dataset_id }
 
     before_save :ensure_valid_datatype
-    after_create :set_date_column_if_date_splitter
     after_save :handle_date_column_change
     before_save :set_defaults
 
@@ -40,6 +41,21 @@ module EasyML
     scope :categorical, -> { where(datatype: %w[categorical string boolean]) }
     scope :datetime, -> { where(datatype: "datetime") }
     scope :date_column, -> { where(is_date_column: true) }
+    scope :required, -> { where(is_computed: false, hidden: false, is_target: false).where("preprocessing_steps IS NULL OR preprocessing_steps::text = '{}'::text") }
+    scope :api_inputs, -> { where(is_computed: false, hidden: false, is_target: false) }
+    scope :computed, -> { where(is_computed: true) }
+
+    def aliases
+      [name].concat(virtual_columns)
+    end
+
+    def virtual_columns
+      if one_hot?
+        allowed_categories.map { |cat| "#{name}_#{cat}" }
+      else
+        []
+      end
+    end
 
     def datatype=(dtype)
       write_attribute(:datatype, dtype)
@@ -88,15 +104,50 @@ module EasyML
     end
 
     def allowed_categories
-      return nil unless one_hot?
+      return [] unless one_hot?
+      stats = dataset.statistics
+      return [] if stats.nil? || stats.blank?
+
+      stats = stats.deep_symbolize_keys
+      stats = stats.dig(:raw)
 
-      dataset.preprocessor.statistics.dup.to_h.dig(name.to_sym, :allowed_categories).sort.concat(["other"])
+      (stats.dig(name.to_sym, :allowed_categories) || []).sort.concat(["other"])
     end
 
     def date_column?
       is_date_column
     end
 
+    def lineage
+      [
+        present_in_raw_dataset ? "Raw dataset" : nil,
+        computed_by ? "Computed by #{computed_by}" : nil,
+        preprocessing_steps.present? ? "Preprocessed using #{preprocessing_steps.keys.join(", ")}" : nil,
+      ].compact
+    end
+
+    def required?
+      is_computed && (preprocessing_steps.nil? || preprocessing_steps == {}) && !hidden && !is_target
+    end
+
+    def present_in_raw_dataset
+      dataset.raw.data&.columns&.include?(name) || false
+    end
+
+    def sort_required
+      required? ? 0 : 1
+    end
+
+    def to_api
+      {
+        name: name,
+        datatype: datatype,
+        description: description,
+        required: required?,
+        allowed_values: allowed_categories.empty? ? nil : allowed_categories,
+      }.compact
+    end
+
     private
 
     def set_defaults

data/app/models/easy_ml/column_history.rb

@@ -9,7 +9,7 @@
 #  description         :string
 #  datatype            :string
 #  polars_datatype     :string
-#  is_target           :boolean
+#  is_target           :boolean          default(FALSE)
 #  hidden              :boolean          default(FALSE)
 #  drop_if_null        :boolean          default(FALSE)
 #  preprocessing_steps :json
@@ -22,10 +22,14 @@
 #  history_user_id     :integer
 #  snapshot_id         :string
 #  is_date_column      :boolean          default(FALSE)
+#  computed_by         :string
+#  is_computed         :boolean          default(FALSE)
 #
 module EasyML
   class ColumnHistory < ActiveRecord::Base
     self.table_name = "easy_ml_column_histories"
     include Historiographer::History
+    scope :required, -> { where(is_computed: false, hidden: false, is_target: false).where("preprocessing_steps IS NULL OR preprocessing_steps::text = '{}'::text") }
+    scope :computed, -> { where(is_computed: true) }
   end
 end

data/app/models/easy_ml/column_list.rb

@@ -1,16 +1,19 @@
 module EasyML
   module ColumnList
-    def sync(only_new: false)
+    include Historiographer::Relation
+
+    def sync(delete: true)
       return unless dataset.schema.present?
 
       EasyML::Column.transaction do
         col_names = syncable
         existing_columns = where(name: col_names)
         import_new(col_names, existing_columns)
+        update_existing(existing_columns)
+        set_feature_lineage
 
-        if !only_new
-          update_existing(existing_columns)
-          delete_missing(existing_columns)
+        if delete
+          delete_missing(col_names)
         end
 
         if existing_columns.none? # Totally new dataset
@@ -37,14 +40,9 @@ module EasyML
       end
     end
 
-    def virtual_column?(column)
-      false
-    end
-
     def syncable
       dataset.processed_schema.keys.select do |col|
-        !one_hot?(col) &&
-          !virtual_column?(col)
+        !one_hot?(col)
       end
     end
 
@@ -56,8 +54,36 @@ module EasyML
       proxy_association.owner
     end
 
+    def sort_by_required
+      column_list.sort_by { |col| [col.sort_required, col.name] }
+    end
+
     private
 
+    def set_feature_lineage
+      # Get all features that compute columns
+      features_computing_columns = dataset.features.all.map do |feature|
+        [feature.name, feature.computes_columns]
+      end.compact.to_h
+
+      updates = column_list.reload.map do |column|
+        # Check if column is computed by any feature
+        computing_feature = features_computing_columns.find { |_, cols| cols.include?(column.name) }&.first
+        is_computed = !computing_feature.nil?
+
+        column.assign_attributes(
+          computed_by: computing_feature,
+          is_computed: is_computed,
+        )
+        next unless column.changed?
+
+        column
+      end.compact
+      EasyML::Column.import(updates.to_a, { on_duplicate_key_update: { columns: %i[computed_by is_computed] } })
+      cols = EasyML::Column.where(id: updates.map(&:id)).to_a
+      column_list.bulk_record_history(cols, { history_user_id: 1 })
+    end
+
     def import_new(new_columns, existing_columns)
       new_columns = new_columns - existing_columns.map(&:name)
       cols_to_insert = new_columns.map do |col_name|
@@ -67,6 +93,7 @@ module EasyML
         )
       end
       EasyML::Column.import(cols_to_insert)
+      column_list.reload
     end
 
     def update_existing(existing_columns)
@@ -116,13 +143,18 @@ module EasyML
       end
       EasyML::Column.import(existing_columns.to_a,
                             { on_duplicate_key_update: { columns: %i[statistics datatype polars_datatype
-                                                                   sample_values] } })
+                                                                   sample_values computed_by is_computed] } })
     end
 
-    def delete_missing(existing_columns)
-      raw_cols = dataset.raw.train(all_columns: true, limit: 1).columns
+    def delete_missing(col_names)
+      raw_cols = dataset.best_segment.train(all_columns: true, limit: 1).columns
       raw_cols = where(name: raw_cols)
-      columns_to_delete = column_list - existing_columns - raw_cols
+      columns_to_delete = column_list.select do |col|
+        col_names.exclude?(col.name) &&
+          one_hots.map(&:name).exclude?(col.name) &&
+          raw_cols.map(&:name).exclude?(col.name) &&
+          dataset.features.flat_map(&:computes_columns).exclude?(col.name)
+      end
       columns_to_delete.each(&:destroy!)
     end
   end

data/app/models/easy_ml/dataset.rb

@@ -140,6 +140,12 @@ module EasyML
       EasyML::RefreshDatasetJob.perform_later(id)
     end
 
+    def best_segment
+      [processed, raw].detect do |segment|
+        segment.send(:train, all_columns: true, limit: 1)&.columns
+      end
+    end
+
     def raw
       return @raw if @raw && @raw.dataset
 
@@ -175,9 +181,10 @@ module EasyML
 
     def actually_refresh
       refreshing do
+        learn(delete: false) # After syncing datasource, learn new statistics + sync columns
         process_data
         fully_reload
-        learn
+        learn # After processing data, we may have new columns from newly applied features
         now = UTC.now
         update(workflow_status: "ready", refreshed_at: now, updated_at: now)
         fully_reload
@@ -272,10 +279,10 @@ module EasyML
       raw.split_at.present? && raw.split_at < datasource.last_updated_at
     end
 
-    def learn(only_new: false)
+    def learn(delete: true)
       learn_schema
       learn_statistics
-      columns.sync(only_new: only_new)
+      columns.sync(delete: delete)
     end
 
     def refreshing
@@ -336,21 +343,25 @@ module EasyML
 
     def learn_statistics
       stats = {
-        raw: EasyML::Data::StatisticsLearner.learn(raw, self),
+        raw: EasyML::Data::StatisticsLearner.learn(raw, self, :raw),
       }
-      stats.merge!(processed: EasyML::Data::StatisticsLearner.learn(processed, self)) if processed.data.present?
+      stats.merge!(processed: EasyML::Data::StatisticsLearner.learn(processed, self, :processed)) if processed.data.present?
+
+      columns.select(&:is_computed).each do |col|
+        if stats.dig(:processed, col.name)
+          stats[:raw][col.name] = stats[:processed][col.name]
+        end
+      end
 
       update(statistics: stats)
     end
 
     def process_data
-      split_data
       fit
       normalize_all
-      # alert_nulls
     end
 
-    def needs_learn?(df)
+    def needs_learn?
       return true if columns_need_refresh?
 
       never_learned = columns.none?
@@ -359,6 +370,7 @@ module EasyML
       new_features = features.any? { |f| f.updated_at > columns.maximum(:updated_at) }
       return true if new_features
 
+      df = raw.query(limit: 1)
       new_cols = df.present? ? (df.columns - columns.map(&:name)) : []
       new_cols = columns.syncable
 
@@ -390,22 +402,24 @@ module EasyML
       { differing_columns: differing_columns, differences: differences }
     end
 
-    def normalize(df = nil, split_ys: false, inference: false, all_columns: false, features: self.features, idx: nil)
-      df = apply_features(df, features)
-      df = drop_nulls(df)
+    def validate_input(df)
+      fields = missing_required_fields(df)
+      return fields.empty?, fields
+    end
+
+    def normalize(df = nil, split_ys: false, inference: false, all_columns: false, features: self.features)
       df = apply_missing_features(df, inference: inference)
+      df = drop_nulls(df)
       df = preprocessor.postprocess(df, inference: inference)
-
-      # Learn will update columns, so if any features have been added
-      # since the last time columns were learned, we should re-learn the schema
-      learn(only_new: true) if idx == 1 && needs_learn?(df)
+      df = apply_features(df, features)
+      learn unless inference # After applying features, we need to learn new statistics
+      df = preprocessor.postprocess(df, inference: inference, computed: true)
       df = apply_column_mask(df, inference: inference) unless all_columns
-      raise_on_nulls(df) if inference
       df, = processed.split_features_targets(df, true, target) if split_ys
       df
     end
 
-    def raise_on_nulls(df)
+    def missing_required_fields(df)
       desc_df = df.describe
 
       # Get the 'null_count' row
@@ -416,8 +430,10 @@ module EasyML
         null_count_row[col][0].to_i > 0
       end
 
-      if columns_with_nulls.any?
-        raise "Null values found in columns: #{columns_with_nulls.join(", ")}"
+      # This is a history class, because this only occurs on prediction
+      required_columns = columns.current.required.map(&:name)
+      required_columns.select do |col|
+        columns_with_nulls.include?(col) || df.columns.map(&:to_s).exclude?(col.to_s)
       end
     end
 
@@ -487,7 +503,7 @@ module EasyML
     end
 
     def preprocessing_steps
-      return if columns.nil? || (columns.respond_to?(:empty?) && columns.empty?)
+      return {} if columns.nil? || (columns.respond_to?(:empty?) && columns.empty?)
       return @preprocessing_steps if @preprocessing_steps.present?
 
       training = standardize_preprocessing_steps(:training)
@@ -515,7 +531,7 @@ module EasyML
     end
 
     def drop_cols
-      @drop_cols ||= preloaded_columns.select(&:hidden).map(&:name)
+      @drop_cols ||= preloaded_columns.select(&:hidden).flat_map(&:aliases)
     end
 
     def drop_if_null
@@ -552,10 +568,14 @@ module EasyML
       df[column_mask(df, inference: inference)]
     end
 
-    def apply_missing_features(df, inference: false)
+    def apply_missing_features(df, inference: false, include_one_hots: false)
       return df unless inference
 
       missing_features = (col_order(inference: inference) - df.columns).compact
+      unless include_one_hots
+        missing_features -= columns.one_hots.flat_map(&:virtual_columns) unless include_one_hots
+        missing_features += columns.one_hots.map(&:name) - df.columns
+      end
       df.with_columns(missing_features.map { |f| Polars.lit(nil).alias(f) })
     end
 
@@ -661,9 +681,9 @@ module EasyML
     def normalize_all
       processed.cleanup
 
-      SPLIT_ORDER.each_with_index do |segment, idx|
+      SPLIT_ORDER.each do |segment|
         df = raw.read(segment)
-        processed_df = normalize(df, all_columns: true, idx: idx)
+        processed_df = normalize(df, all_columns: true)
         processed.save(segment, processed_df)
       end
       @normalized = true
@@ -687,8 +707,9 @@ module EasyML
     end
 
     def fit
-      preprocessor.fit(raw.train(all_columns: true))
-      self.preprocessor_statistics = preprocessor.statistics
+      computed_statistics = columns.where(is_computed: true).reduce({}) { |h, c| h.tap { h[c.name] = c.statistics.dig("processed") } }
+      preprocessor.fit(raw.train(all_columns: true), computed_statistics)
+      update(preprocessor_statistics: preprocessor.statistics)
     end
 
     # log_method :fit, "Learning statistics", verbose: true
@@ -701,7 +722,6 @@ module EasyML
       return unless force || should_split?
 
       cleanup
-      features = self.features.ordered.load
       splitter.split(datasource) do |train_df, valid_df, test_df|
         [:train, :valid, :test].zip([train_df, valid_df, test_df]).each do |segment, df|
           raw.save(segment, df)

data/app/models/easy_ml/datasource.rb

@@ -55,6 +55,7 @@ module EasyML
 
     has_many :events, as: :eventable, class_name: "EasyML::Event", dependent: :destroy
     attr_accessor :schema, :columns, :num_rows, :is_syncing
+    belongs_to :dataset, class_name: "EasyML::Dataset", optional: true, dependent: :destroy
 
     add_configuration_attributes :schema, :columns, :num_rows, :polars_args, :verbose, :is_syncing
     DATASOURCE_CONSTANTS.flat_map(&:configuration_attributes).each do |attribute|

data/app/models/easy_ml/feature.rb

@@ -165,6 +165,13 @@ module EasyML
       end
     end
 
+    def computes_columns
+      unless adapter.respond_to?(:computes_columns)
+        raise "Feature #{feature_class} must declare which columns it computes using the :computes_columns method"
+      end
+      adapter.computes_columns
+    end
+
     def build_batches
       if batchable?
         batch
@@ -239,7 +246,7 @@ module EasyML
 
     # Transform a single batch, used for testing the user's feature implementation
     def transform_batch(df = nil, batch_args = {})
-      if df.present?
+      if df.is_a?(Polars::DataFrame)
         actually_transform_batch(df)
       else
         actually_transform_batch(build_batch(get_batch_args(**batch_args)))
@@ -296,8 +303,8 @@ module EasyML
     end
 
     def actually_transform_batch(df)
-      return nil unless df.present?
-      return df if adapter.respond_to?(:fit) && feature_store.empty?
+      return nil unless df.is_a?(Polars::DataFrame)
+      return df if !adapter.respond_to?(:transform) && feature_store.empty?
 
       result = adapter.transform(df, self)
       update!(applied_at: Time.current)