easy_ml 0.2.0.pre.rc101 → 0.2.0.pre.rc103

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f48937aea567de8e40bc34486c4ac945b860ca26654d8d3b06efa1c1d4a54f3
-  data.tar.gz: 1abb8bb2e3f3ba8bb9c228d7a9691e8906ababa523ad0d7155cdafbd3ec62396
+  metadata.gz: ef3f840cce99d7205957fbb39a6b319a45035624dce2e4e10f681383cb088abf
+  data.tar.gz: e25100f792ad48cfa4feab7eb652a2d6c49bfc6e28f3bcb97c8150f9bdd1bfc5
 SHA512:
-  metadata.gz: ef28fcb989d2934329e4da3c9a138d3fc7b4c9ae995d7ce021217f4507e24b17664d2dee9690b7a00105e804b41f490442e2b90f1d76f8c12d7ddca768ae43ba
-  data.tar.gz: 3ba6f95ca3a660540e81a49c5eba84f530b606d89dc499e05aa288d26b90802dfc74b0b7615360002f1f36c1255f41ea6871378ffa1e93abc147b4f2a5c6ab0c
+  metadata.gz: 5f58395d392158d149db34ad5019a0e011164ca8d331846553e44e6564a291d88323ad0090c1c5ded60f696940b30949cba4e1a614fa9cd502e94372ef949707
+  data.tar.gz: 9497391351ad054308a985cc6b9e608f8dfef61be7417d66502cb11c26ca4f7825456b31aab010c016ac10feff791a8f7a01893743ecf11a26fabb9de7405b82

data/app/controllers/easy_ml/datasets_controller.rb

@@ -190,6 +190,7 @@ module EasyML
         :description,
         :datasource_id,
         :target,
+        :view_class,
         drop_cols: [],
         splitter_attributes: %i[
           splitter_type

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -1028,7 +1028,6 @@ export function PreprocessingConfig({
                       label: strategy.label
                     })) || [])
                   ]}
-                  options={constants.preprocessing_strategies[selectedType]}
                 />
 
                 {renderStrategySpecificInfo('training')}

data/app/frontend/components/dataset/splitters/types.ts

@@ -1,12 +1,11 @@
-import type { ColumnType } from '../../../types/datasource';
+import type { Constants } from '../../../types/datasource';
 import type { Datasource } from '../types/datasource';
 
 export type NewDatasetFormProps = {
     datasources: Datasource[];
-    constants: {
-        columns: ColumnType[];
-    };
+    constants: Constants;
 }
+
 export type SplitterType = 
   | 'date'
   | 'random'

data/app/frontend/pages/NewDatasetPage.tsx

@@ -78,6 +78,7 @@ export default function NewDatasetPage({ constants, datasources }: NewDatasetFor
     dataset: {
       name: '',
       datasource_id: '',
+      view_class: '',
       splitter_attributes: {
         splitter_type: selectedSplitterType,
         ...getDefaultConfig(selectedSplitterType)
@@ -249,6 +250,22 @@ export default function NewDatasetPage({ constants, datasources }: NewDatasetFor
               />
             </div>
 
+            <div>
+              <label
+                htmlFor="view_class"
+                className="block text-sm font-medium text-gray-700"
+              >
+                View Class
+              </label>
+              <SearchableSelect
+                value={formData.dataset.view_class}
+                onChange={(value) => setData('dataset.view_class', value)}
+                options={constants.available_views}
+                className="mt-1"
+                placeholder="Select a view class (optional)..."
+              />
+            </div>
+
             {selectedDatasource && (
               <div className={`rounded-lg p-4 ${
                 selectedDatasource.sync_error 

data/app/frontend/types/datasource.ts

@@ -10,6 +10,19 @@ export interface Schema {
   [key: string]: ColumnType;
 }
 
+export interface Constants {
+  column_types: Array<{ value: string; label: string }>;
+  preprocessing_strategies: any;
+  feature_options: any;
+  splitter_constants: any;
+  embedding_constants: any;
+  available_views: Array<{ value: string; label: string }>;
+  DATASOURCE_TYPES: Array<{ value: string; label: string; description: string }>;
+  s3: {
+    S3_REGIONS: Array<{ value: string; label: string }>;
+  };
+}
+
 export interface Datasource {
   id: number;
   name: string;
@@ -23,10 +36,5 @@ export interface Datasource {
 
 export interface DatasourceFormProps {
   datasource?: Datasource;
-  constants: {
-    DATASOURCE_TYPES: Array<{ value: string; label: string; description: string }>;
-    s3: {
-      S3_REGIONS: Array<{ value: string; label: string }>;
-    };
-  };
+  constants: Constants;
 }

data/app/models/easy_ml/column/imputers/base.rb

@@ -46,6 +46,8 @@ module EasyML
         end
 
         def param_applies?
+          return false unless params.present?
+
           params.keys.any? { |p| imputers_own_params.include?(p.to_sym) && params[p] != false }
         end
 
@@ -60,7 +62,7 @@ module EasyML
         end
 
         def imputers_own_params
-          Imputers.params_by_class[self.class] || []
+          Imputers.params_by_class[self.class] || {}
         end
 
         def imputers_own_encodings

data/app/models/easy_ml/column.rb

@@ -71,6 +71,7 @@ module EasyML
     scope :has_clip, -> { where("preprocessing_steps->'training'->>'params' IS NOT NULL AND preprocessing_steps->'training'->'params' @> jsonb_build_object('clip', jsonb_build_object())") }
     scope :needs_learn, -> {
             datasource_changed
+              .or(is_view)
               .or(feature_applied)
               .or(feature_changed)
               .or(column_changed)
@@ -88,6 +89,13 @@ module EasyML
               )
           }
 
+    scope :is_view, -> { 
+      left_joins(dataset: :datasource)
+          .left_joins(:feature)
+          .where(
+            Dataset.arel_table[:view_class].not_eq(nil)
+          )
+    }
     scope :feature_changed, -> {
             where(feature_id: Feature.has_changes.map(&:id))
           }
@@ -514,27 +522,32 @@ module EasyML
       EasyML::Import::Column.from_config(config, dataset, action: action)
     end
 
-    def cast_statement(df, df_col, expected_dtype)
-      expected_dtype = expected_dtype.is_a?(Polars::DataType) ? expected_dtype.class : expected_dtype
-      actual_type = df[df_col].dtype
+    def cast_statement(series = nil)
+      expected_dtype = polars_datatype
+      actual_type = series&.dtype || expected_dtype
+
+      return Polars.col(name).cast(expected_dtype).alias(name) if expected_dtype == actual_type
 
       cast_statement = case expected_dtype.to_s
-                       when "Polars::Boolean"
+                        when /Polars::List/
+                          # we should start tracking polars args so we can know what type of list it is
+                          Polars.col(name)
+                        when /Polars::Boolean/
                           case actual_type.to_s
-                          when "Polars::Boolean"
-                            Polars.col(df_col).cast(expected_dtype)
-                          when "Polars::Utf8", "Polars::Categorical", "Polars::String"
-                            Polars.col(df_col).eq("true").cast(expected_dtype)
-                          when "Polars::Null"
-                            Polars.col(df_col)
+                          when /Polars::Boolean/, /Polars::Int/
+                            Polars.col(name).cast(expected_dtype)
+                          when /Polars::Utf/, /Polars::Categorical/, /Polars::String/
+                            Polars.col(name).eq("true").cast(expected_dtype)
+                          when /Polars::Null/
+                            Polars.col(name)
                           else
-                            raise "Unexpected dtype: #{actual_type} for column: #{df_col}"
+                            raise "Unexpected dtype: #{actual_type} for column: #{name}"
                           end
                         else
-                          Polars.col(df_col).cast(expected_dtype)
+                          Polars.col(name).cast(expected_dtype, strict: false)
                         end
 
-      cast_statement.alias(df_col)
+      cast_statement.alias(name)
     end
 
     def cast(value)

data/app/models/easy_ml/column_list.rb

@@ -101,10 +101,10 @@ module EasyML
       end
       cast_statements = (df.columns & schema.keys.map(&:to_s)).map do |df_col|
         db_col = column_index[df_col]
-        expected_dtype = schema[df_col.to_sym]
-        db_col.cast_statement(df, df_col, expected_dtype)
+        db_col.cast_statement(df[df_col])
       end
       df = df.with_columns(cast_statements)
+      df
     end
 
     def cast(processed_or_raw)

data/app/models/easy_ml/dataset/learner/lazy/datetime.rb

@@ -10,7 +10,9 @@ module EasyML
           end
 
           def unique_count
-            Polars.col(column.name).n_unique.alias("#{column.name}__unique_count")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .n_unique.alias("#{column.name}__unique_count")
           end
         end
       end

data/app/models/easy_ml/dataset/learner/lazy/numeric.rb

@@ -5,11 +5,30 @@ module EasyML
         class Numeric < Query
           def train_query
             super.concat([
-              Polars.col(column.name).mean.alias("#{column.name}__mean"),
-              Polars.col(column.name).median.alias("#{column.name}__median"),
-              Polars.col(column.name).min.alias("#{column.name}__min"),
-              Polars.col(column.name).max.alias("#{column.name}__max"),
-              Polars.col(column.name).std.alias("#{column.name}__std"),
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .mean
+                    .alias("#{column.name}__mean"),
+
+              Polars.col(column.name)
+                      .cast(column.polars_datatype)
+                      .median
+                      .alias("#{column.name}__median"),
+
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .min
+                    .alias("#{column.name}__min"),
+
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .max
+                    .alias("#{column.name}__max"),
+
+              Polars.col(column.name)
+                    .cast(column.polars_datatype)
+                    .std
+                    .alias("#{column.name}__std"),
             ])
           end
         end

data/app/models/easy_ml/dataset/learner/lazy/query.rb

@@ -44,25 +44,37 @@ module EasyML
           end
 
           def null_count
-            Polars.col(column.name).null_count.alias("#{column.name}__null_count")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .null_count
+                  .alias("#{column.name}__null_count")
           end
 
           def num_rows
-            Polars.col(column.name).len.alias("#{column.name}__num_rows")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .len
+                  .alias("#{column.name}__num_rows")
           end
 
           def most_frequent_value
-            Polars.col(column.name).filter(Polars.col(column.name).is_not_null).mode.first.alias("#{column.name}__most_frequent_value")
+            Polars.col(column.name)
+                  .cast(column.polars_datatype)
+                  .filter(Polars.col(column.name).is_not_null)
+                  .mode
+                  .first
+                  .alias("#{column.name}__most_frequent_value")
           end
 
           def last_value
             return unless dataset.date_column.present?
 
             Polars.col(column.name)
-              .sort_by(dataset.date_column.name, reverse: true, nulls_last: true)
-              .filter(Polars.col(column.name).is_not_null)
-              .first
-              .alias("#{column.name}__last_value")
+                  .cast(column.polars_datatype)
+                  .sort_by(dataset.date_column.name, reverse: true, nulls_last: true)
+                  .filter(Polars.col(column.name).is_not_null)
+                  .first
+                  .alias("#{column.name}__last_value")
           end
         end
       end

data/app/models/easy_ml/dataset/learner/lazy/string.rb

@@ -10,7 +10,10 @@ module EasyML
           end
 
           def unique_count
-            Polars.col(column.name).cast(:str).n_unique.alias("#{column.name}__unique_count")
+            Polars.col(column.name)
+                  .cast(Polars::String)
+                  .n_unique
+                  .alias("#{column.name}__unique_count")
           end
         end
       end

data/app/models/easy_ml/dataset/learner/lazy.rb

@@ -22,9 +22,22 @@ module EasyML
         def run_queries(split, type)
           queries = build_queries(split, type)
 
-          dataset.columns.apply_clip(
-            @dataset.send(type).send(split, all_columns: true, lazy: true)
-          ).select(queries).collect
+          begin
+            dataset.columns.apply_clip(
+              @dataset.send(type).send(split, all_columns: true, lazy: true)
+            )
+            .select(queries).collect
+          rescue => e
+            problematic_queries = queries.select { |query| 
+              begin
+                dataset.send(type).send(split, all_columns: true, lazy: true).select([query]).collect
+                false
+              rescue => e
+                true
+              end
+            }
+            raise "Query failed for queries... likely due to wrong column datatype: #{problematic_queries.join("\n")}"
+          end
         end
 
         def get_column_statistics(query_results)
@@ -51,4 +64,4 @@ module EasyML
       end
     end
   end
-end
+end

data/app/models/easy_ml/dataset.rb

@@ -20,6 +20,7 @@
 #  updated_at          :datetime         not null
 #  last_datasource_sha :string
 #  raw_schema          :jsonb
+#  view_class          :string
 #
 module EasyML
   class Dataset < ActiveRecord::Base
@@ -64,6 +65,7 @@ module EasyML
                                   reject_if: :all_blank
 
     validates :datasource, presence: true
+    validate :view_class_exists, if: -> { view_class.present? }
 
     add_configuration_attributes :remote_files
 
@@ -85,6 +87,10 @@ module EasyML
         feature_options: EasyML::Features::Registry.list_flat,
         splitter_constants: EasyML::Splitter.constants,
         embedding_constants: EasyML::Data::Embeddings::Embedder.constants,
+        available_views: Rails.root.join("app/datasets").glob("*.rb").map { |f| 
+          name = f.basename(".rb").to_s.camelize
+          { value: name, label: name.titleize }
+        }
       }
     end
 
@@ -148,7 +154,7 @@ module EasyML
       return @schema if @schema
       return read_attribute(:schema) if @serializing
 
-      schema = read_attribute(:schema) || datasource.schema || datasource.after_sync.schema
+      schema = read_attribute(:schema) || materialized_view&.schema || datasource.schema || datasource.after_sync.schema
       schema = set_schema(schema)
       @schema = EasyML::Data::PolarsSchema.deserialize(schema)
     end
@@ -157,7 +163,7 @@ module EasyML
       return @raw_schema if @raw_schema
       return read_attribute(:raw_schema) if @serializing
 
-      raw_schema = read_attribute(:raw_schema) || datasource.schema || datasource.after_sync.schema
+      raw_schema = read_attribute(:raw_schema) || materialized_view&.schema || datasource.schema || datasource.after_sync.schema
       raw_schema = set_raw_schema(raw_schema)
       @raw_schema = EasyML::Data::PolarsSchema.deserialize(raw_schema)
     end
@@ -178,7 +184,12 @@ module EasyML
       if datasource&.num_rows.nil?
         datasource.after_sync
       end
-      datasource&.num_rows
+
+      if materialized_view.present?
+        materialized_view.shape[0]
+      else
+        datasource&.num_rows
+      end
     end
 
     def abort!
@@ -234,6 +245,29 @@ module EasyML
       features.update_all(workflow_status: "ready")
     end
 
+    def view_class_exists
+      begin
+        view_class.constantize
+      rescue NameError
+        errors.add(:view_class, "must be a valid class name")
+      end
+    end
+
+    def materialize_view(df)
+      df
+    end
+
+    def materialized_view
+      return @materialized_view if @materialized_view
+
+      original_df = datasource.data
+      if view_class.present?
+        @materialized_view = view_class.constantize.new.materialize_view(original_df)
+      else
+        @materialized_view = materialize_view(original_df)
+      end
+    end
+
     def prepare!
       prepare_features
       cleanup
@@ -423,6 +457,7 @@ module EasyML
     end
 
     def needs_learn?
+      return true if view_class.present?
       return true if columns_need_refresh?
 
       never_learned = columns.none?
@@ -471,6 +506,7 @@ module EasyML
     def normalize(df = nil, split_ys: false, inference: false, all_columns: false, features: self.features)
       df = apply_missing_columns(df, inference: inference)
       df = transform_columns(df, inference: inference, encode: false)
+      df = apply_cast(df)
       df = apply_features(df, features, inference: inference)
       df = apply_cast(df) if inference
       df = transform_columns(df, inference: inference)
@@ -798,7 +834,8 @@ module EasyML
       df = df.clone
       df = apply_features(df)
       processed.save(:train, df)
-      learn_statistics(type: :processed)
+      learn(delete: false)
+      learn_statistics(type: :processed, computed: true)
       processed.cleanup
     end
 
@@ -836,11 +873,12 @@ module EasyML
       return unless force || needs_refresh?
 
       cleanup
-      splitter.split(datasource) do |train_df, valid_df, test_df|
-        [:train, :valid, :test].zip([train_df, valid_df, test_df]).each do |segment, df|
-          raw.save(segment, df)
-        end
-      end
+
+      train_df, valid_df, test_df = splitter.split(self)
+      raw.save(:train, train_df)
+      raw.save(:valid, valid_df)
+      raw.save(:test, test_df)
+
       raw_schema # Set if not already set
     end
 

data/app/models/easy_ml/dataset_history.rb

@@ -25,6 +25,7 @@
 #  snapshot_id         :string
 #  last_datasource_sha :string
 #  raw_schema          :jsonb
+#  view_class          :string
 #
 module EasyML
   class DatasetHistory < ActiveRecord::Base

data/app/models/easy_ml/feature.rb

@@ -277,24 +277,16 @@ module EasyML
         feature.fit_batch(batch_args.merge!(batch_id: batch_id))
       rescue => e
         EasyML::Feature.transaction do
-          return if dataset.reload.workflow_status == :failed
-
-          feature.update(workflow_status: :failed)
-          dataset.update(workflow_status: :failed)
-          build_error_with_context(dataset, e, batch_id, feature)
+          if dataset.reload.workflow_status != :failed
+            feature.update(workflow_status: :failed)
+            dataset.update(workflow_status: :failed)
+            EasyML::Event.handle_error(dataset, e)
+          end
         end
         raise e
       end
     end
 
-    def self.build_error_with_context(dataset, error, batch_id, feature)
-      error = EasyML::Event.handle_error(dataset, error)
-      batch = feature.build_batch(batch_id: batch_id)
-
-      # Convert any dataframes in the context to serialized form
-      error.create_context(context: batch)
-    end
-
     def self.fit_feature_failed(dataset, e)
       dataset.update(workflow_status: :failed)
       EasyML::Event.handle_error(dataset, e)

data/app/models/easy_ml/lineage.rb

@@ -31,12 +31,13 @@ module EasyML
         }
         existing_lineage = existing_lineage.map do |key, lineage|
           matching_lineage = @lineage.detect { |ll| ll[:key].to_sym == lineage.key.to_sym }
+          next unless matching_lineage.present?
 
           lineage&.assign_attributes(
             occurred_at: matching_lineage[:occurred_at],
             description: matching_lineage[:description],
           )
-        end
+        end.compact
         missing_lineage.concat(existing_lineage)
       end
     end

data/app/models/easy_ml/models/xgboost/evals_callback.rb

@@ -40,6 +40,7 @@ module EasyML
           end
         end
 
+        # STOP CHECKING S3 IN BETWEEN ITERATIONS... FIND WHERE REFRESH IS GETTING CALLED
         def after_iteration(booster, epoch, history)
           return false unless wandb_enabled?
 

data/app/models/easy_ml/models/xgboost.rb

@@ -320,7 +320,10 @@ module EasyML
         raise "Cannot predict on nil — XGBoost" if xs.nil?
 
         begin
+          @predicting = true
           y_pred = yield(preprocess(xs))
+          @predicting = false
+          y_pred
         rescue StandardError => e
           raise e unless e.message.match?(/Number of columns does not match/)
 
@@ -495,12 +498,14 @@ module EasyML
         feature_cols -= [weights_col] if weights_col
 
         # Get features, labels and weights
-        exploded = explode_embeddings(xs.select(feature_cols))
+        exploded = explode_embeddings(xs)
         feature_cols = exploded.columns
         features = lazy ? exploded.collect.to_numo : exploded.to_numo
 
-        weights = weights_col ? (lazy ? xs.select(weights_col).collect.to_numo : xs.select(weights_col).to_numo) : nil
-        weights = weights.flatten if weights
+        unless @predicting
+          weights = weights_col ? (lazy ? xs.select(weights_col).collect.to_numo : xs.select(weights_col).to_numo) : nil
+          weights = weights.flatten if weights
+        end
         if ys.present?
           ys = ys.is_a?(Array) ? Polars::Series.new(ys) : ys
           labels = lazy ? ys.collect.to_numo.flatten : ys.to_numo.flatten

data/app/models/easy_ml/prediction.rb

@@ -28,7 +28,7 @@ module EasyML
 
     def prediction
       prediction_value["value"]
-    end
+e   end
 
     def probabilities
       metadata["probabilities"]

data/app/models/easy_ml/splitters/base_splitter.rb

@@ -6,18 +6,14 @@ module EasyML
 
       attr_reader :splitter
 
-      def split(datasource, &block)
-        datasource.in_batches do |df|
-          split_df(df).tap do |splits|
-            yield splits if block_given?
-          end
-        end
-      end
-
       def split_df(df)
         df
       end
 
+      def split(dataset)
+        split_df(dataset.materialized_view)
+      end
+
       def initialize(splitter)
         @splitter = splitter
       end

data/app/models/easy_ml/splitters/date_splitter.rb

@@ -41,9 +41,10 @@ module EasyML
 
         validation_date_start, test_date_start = splits
 
+        dtype = df[date_col].dtype
         test_df = Polars.concat(
           [
-            df.filter(Polars.col(date_col) >= test_date_start),
+            df.filter(Polars.col(date_col).ge(Polars.lit(test_date_start).cast(dtype))),
             df.filter(Polars.col(date_col).is_null),
           ]
         )

data/app/models/easy_ml/splitters/predefined_splitter.rb

@@ -15,13 +15,18 @@ module EasyML
         }
       end
 
-      def split(datasource, &block)
+      def split(dataset, &block)
         validate!
 
-        files = datasource.all_files
+        files = dataset.datasource.all_files
         train, valid, test = match_files(files)
 
-        yield [reader.query(train), reader.query(valid), reader.query(test)]
+        values = [reader.query(train), reader.query(valid), reader.query(test)]
+        if block_given?
+          yield values
+        else
+          values
+        end
       end
 
       def match_files(files)