easy_ml 0.2.0.pre.rc101 → 0.2.0.pre.rc102

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f48937aea567de8e40bc34486c4ac945b860ca26654d8d3b06efa1c1d4a54f3
-  data.tar.gz: 1abb8bb2e3f3ba8bb9c228d7a9691e8906ababa523ad0d7155cdafbd3ec62396
+  metadata.gz: 4c4736c4959fd2d08faac5dbe0c4525014edb8faa7e5b914875a0a84f58e53f2
+  data.tar.gz: bbab12ed80cf8c3bd608388648cd8362d7f4b46408b135aaf79ef494dca7deed
 SHA512:
-  metadata.gz: ef28fcb989d2934329e4da3c9a138d3fc7b4c9ae995d7ce021217f4507e24b17664d2dee9690b7a00105e804b41f490442e2b90f1d76f8c12d7ddca768ae43ba
-  data.tar.gz: 3ba6f95ca3a660540e81a49c5eba84f530b606d89dc499e05aa288d26b90802dfc74b0b7615360002f1f36c1255f41ea6871378ffa1e93abc147b4f2a5c6ab0c
+  metadata.gz: 8dd7645d2b4da2d03a0c3fc1eaf9bcfdfd05ae31e9871782154ade2149ca4269ee5d78e6cb959d6f10e498cdbd427dfb958dc37c3b1208b0fe8885abac61dcad
+  data.tar.gz: 6690b85ba40db78063ffe8fbf3b9302b82a52c3137f5a59d8ba8bb70838ebd931965968185b6031b90fa0b9d8f3192bcc1d2150e3cce43a5ad1959af738d180e

data/app/controllers/easy_ml/datasets_controller.rb

@@ -190,6 +190,7 @@ module EasyML
         :description,
         :datasource_id,
         :target,
+        :view_class,
         drop_cols: [],
         splitter_attributes: %i[
           splitter_type

data/app/frontend/components/dataset/splitters/types.ts

@@ -1,12 +1,11 @@
-import type { ColumnType } from '../../../types/datasource';
+import type { Constants } from '../../../types/datasource';
 import type { Datasource } from '../types/datasource';
 
 export type NewDatasetFormProps = {
     datasources: Datasource[];
-    constants: {
-        columns: ColumnType[];
-    };
+    constants: Constants;
 }
+
 export type SplitterType = 
   | 'date'
   | 'random'

data/app/frontend/pages/NewDatasetPage.tsx

@@ -78,6 +78,7 @@ export default function NewDatasetPage({ constants, datasources }: NewDatasetFor
     dataset: {
       name: '',
       datasource_id: '',
+      view_class: '',
       splitter_attributes: {
         splitter_type: selectedSplitterType,
         ...getDefaultConfig(selectedSplitterType)
@@ -249,6 +250,22 @@ export default function NewDatasetPage({ constants, datasources }: NewDatasetFor
               />
             </div>
 
+            <div>
+              <label
+                htmlFor="view_class"
+                className="block text-sm font-medium text-gray-700"
+              >
+                View Class
+              </label>
+              <SearchableSelect
+                value={formData.dataset.view_class}
+                onChange={(value) => setData('dataset.view_class', value)}
+                options={constants.available_views}
+                className="mt-1"
+                placeholder="Select a view class (optional)..."
+              />
+            </div>
+
             {selectedDatasource && (
               <div className={`rounded-lg p-4 ${
                 selectedDatasource.sync_error 

data/app/frontend/types/datasource.ts

@@ -10,6 +10,19 @@ export interface Schema {
   [key: string]: ColumnType;
 }
 
+export interface Constants {
+  column_types: Array<{ value: string; label: string }>;
+  preprocessing_strategies: any;
+  feature_options: any;
+  splitter_constants: any;
+  embedding_constants: any;
+  available_views: Array<{ value: string; label: string }>;
+  DATASOURCE_TYPES: Array<{ value: string; label: string; description: string }>;
+  s3: {
+    S3_REGIONS: Array<{ value: string; label: string }>;
+  };
+}
+
 export interface Datasource {
   id: number;
   name: string;
@@ -23,10 +36,5 @@ export interface Datasource {
 
 export interface DatasourceFormProps {
   datasource?: Datasource;
-  constants: {
-    DATASOURCE_TYPES: Array<{ value: string; label: string; description: string }>;
-    s3: {
-      S3_REGIONS: Array<{ value: string; label: string }>;
-    };
-  };
+  constants: Constants;
 }

data/app/models/easy_ml/column/imputers/base.rb

@@ -46,6 +46,8 @@ module EasyML
         end
 
         def param_applies?
+          return false unless params.present?
+
           params.keys.any? { |p| imputers_own_params.include?(p.to_sym) && params[p] != false }
         end
 
@@ -60,7 +62,7 @@ module EasyML
         end
 
         def imputers_own_params
-          Imputers.params_by_class[self.class] || []
+          Imputers.params_by_class[self.class] || {}
         end
 
         def imputers_own_encodings

data/app/models/easy_ml/column.rb

@@ -71,6 +71,7 @@ module EasyML
     scope :has_clip, -> { where("preprocessing_steps->'training'->>'params' IS NOT NULL AND preprocessing_steps->'training'->'params' @> jsonb_build_object('clip', jsonb_build_object())") }
     scope :needs_learn, -> {
             datasource_changed
+              .or(is_view)
               .or(feature_applied)
               .or(feature_changed)
               .or(column_changed)
@@ -88,6 +89,13 @@ module EasyML
               )
           }
 
+    scope :is_view, -> { 
+      left_joins(dataset: :datasource)
+          .left_joins(:feature)
+          .where(
+            Dataset.arel_table[:view_class].not_eq(nil)
+          )
+    }
     scope :feature_changed, -> {
             where(feature_id: Feature.has_changes.map(&:id))
           }

data/app/models/easy_ml/dataset/learner/lazy.rb

@@ -22,9 +22,22 @@ module EasyML
         def run_queries(split, type)
           queries = build_queries(split, type)
 
-          dataset.columns.apply_clip(
-            @dataset.send(type).send(split, all_columns: true, lazy: true)
-          ).select(queries).collect
+          begin
+            dataset.columns.apply_clip(
+              @dataset.send(type).send(split, all_columns: true, lazy: true)
+            )
+            .select(queries).collect
+          rescue => e
+            problematic_query = queries.detect { 
+              begin
+                dataset.send(type).send(split, all_columns: true, lazy: true).select(queries).collect
+                false
+              rescue => e
+                true
+              end
+            }
+            raise "Query failed for column #{problematic_query}, likely wrong datatype"
+          end
         end
 
         def get_column_statistics(query_results)

data/app/models/easy_ml/dataset.rb

@@ -20,6 +20,7 @@
 #  updated_at          :datetime         not null
 #  last_datasource_sha :string
 #  raw_schema          :jsonb
+#  view_class          :string
 #
 module EasyML
   class Dataset < ActiveRecord::Base
@@ -64,6 +65,7 @@ module EasyML
                                   reject_if: :all_blank
 
     validates :datasource, presence: true
+    validate :view_class_exists, if: -> { view_class.present? }
 
     add_configuration_attributes :remote_files
 
@@ -85,6 +87,10 @@ module EasyML
         feature_options: EasyML::Features::Registry.list_flat,
         splitter_constants: EasyML::Splitter.constants,
         embedding_constants: EasyML::Data::Embeddings::Embedder.constants,
+        available_views: Rails.root.join("app/datasets").glob("*.rb").map { |f| 
+          name = f.basename(".rb").to_s.camelize
+          { value: name, label: name.titleize }
+        }
       }
     end
 
@@ -148,7 +154,7 @@ module EasyML
       return @schema if @schema
       return read_attribute(:schema) if @serializing
 
-      schema = read_attribute(:schema) || datasource.schema || datasource.after_sync.schema
+      schema = read_attribute(:schema) || materialized_view&.schema || datasource.schema || datasource.after_sync.schema
       schema = set_schema(schema)
       @schema = EasyML::Data::PolarsSchema.deserialize(schema)
     end
@@ -157,7 +163,7 @@ module EasyML
       return @raw_schema if @raw_schema
       return read_attribute(:raw_schema) if @serializing
 
-      raw_schema = read_attribute(:raw_schema) || datasource.schema || datasource.after_sync.schema
+      raw_schema = read_attribute(:raw_schema) || materialized_view&.schema || datasource.schema || datasource.after_sync.schema
       raw_schema = set_raw_schema(raw_schema)
       @raw_schema = EasyML::Data::PolarsSchema.deserialize(raw_schema)
     end
@@ -178,7 +184,12 @@ module EasyML
       if datasource&.num_rows.nil?
         datasource.after_sync
       end
-      datasource&.num_rows
+
+      if materialized_view.present?
+        materialized_view.shape[0]
+      else
+        datasource&.num_rows
+      end
     end
 
     def abort!
@@ -234,6 +245,29 @@ module EasyML
       features.update_all(workflow_status: "ready")
     end
 
+    def view_class_exists
+      begin
+        view_class.constantize
+      rescue NameError
+        errors.add(:view_class, "must be a valid class name")
+      end
+    end
+
+    def materialize_view(df)
+      df
+    end
+
+    def materialized_view
+      return @materialized_view if @materialized_view
+
+      original_df = datasource.data
+      if view_class.present?
+        @materialized_view = view_class.constantize.new.materialize_view(original_df)
+      else
+        @materialized_view = materialize_view(original_df)
+      end
+    end
+
     def prepare!
       prepare_features
       cleanup
@@ -423,6 +457,7 @@ module EasyML
     end
 
     def needs_learn?
+      return true if view_class.present?
       return true if columns_need_refresh?
 
       never_learned = columns.none?
@@ -471,6 +506,7 @@ module EasyML
     def normalize(df = nil, split_ys: false, inference: false, all_columns: false, features: self.features)
       df = apply_missing_columns(df, inference: inference)
       df = transform_columns(df, inference: inference, encode: false)
+      df = apply_cast(df)
       df = apply_features(df, features, inference: inference)
       df = apply_cast(df) if inference
       df = transform_columns(df, inference: inference)
@@ -798,7 +834,8 @@ module EasyML
       df = df.clone
       df = apply_features(df)
       processed.save(:train, df)
-      learn_statistics(type: :processed)
+      learn(delete: false)
+      learn_statistics(type: :processed, computed: true)
       processed.cleanup
     end
 
@@ -836,11 +873,12 @@ module EasyML
       return unless force || needs_refresh?
 
       cleanup
-      splitter.split(datasource) do |train_df, valid_df, test_df|
-        [:train, :valid, :test].zip([train_df, valid_df, test_df]).each do |segment, df|
-          raw.save(segment, df)
-        end
-      end
+
+      train_df, valid_df, test_df = splitter.split(self)
+      raw.save(:train, train_df)
+      raw.save(:valid, valid_df)
+      raw.save(:test, test_df)
+
       raw_schema # Set if not already set
     end
 

data/app/models/easy_ml/dataset_history.rb

@@ -25,6 +25,7 @@
 #  snapshot_id         :string
 #  last_datasource_sha :string
 #  raw_schema          :jsonb
+#  view_class          :string
 #
 module EasyML
   class DatasetHistory < ActiveRecord::Base

data/app/models/easy_ml/feature.rb

@@ -277,24 +277,16 @@ module EasyML
         feature.fit_batch(batch_args.merge!(batch_id: batch_id))
       rescue => e
         EasyML::Feature.transaction do
-          return if dataset.reload.workflow_status == :failed
-
-          feature.update(workflow_status: :failed)
-          dataset.update(workflow_status: :failed)
-          build_error_with_context(dataset, e, batch_id, feature)
+          if dataset.reload.workflow_status != :failed
+            feature.update(workflow_status: :failed)
+            dataset.update(workflow_status: :failed)
+            EasyML::Event.handle_error(dataset, e)
+          end
         end
         raise e
       end
     end
 
-    def self.build_error_with_context(dataset, error, batch_id, feature)
-      error = EasyML::Event.handle_error(dataset, error)
-      batch = feature.build_batch(batch_id: batch_id)
-
-      # Convert any dataframes in the context to serialized form
-      error.create_context(context: batch)
-    end
-
     def self.fit_feature_failed(dataset, e)
       dataset.update(workflow_status: :failed)
       EasyML::Event.handle_error(dataset, e)

data/app/models/easy_ml/lineage.rb

@@ -31,12 +31,13 @@ module EasyML
         }
         existing_lineage = existing_lineage.map do |key, lineage|
           matching_lineage = @lineage.detect { |ll| ll[:key].to_sym == lineage.key.to_sym }
+          next unless matching_lineage.present?
 
           lineage&.assign_attributes(
             occurred_at: matching_lineage[:occurred_at],
             description: matching_lineage[:description],
           )
-        end
+        end.compact
         missing_lineage.concat(existing_lineage)
       end
     end

data/app/models/easy_ml/models/xgboost/evals_callback.rb

@@ -40,6 +40,7 @@ module EasyML
           end
         end
 
+        # STOP CHECKING S3 IN BETWEEN ITERATIONS... FIND WHERE REFRESH IS GETTING CALLED
         def after_iteration(booster, epoch, history)
           return false unless wandb_enabled?
 

data/app/models/easy_ml/models/xgboost.rb

@@ -320,7 +320,10 @@ module EasyML
         raise "Cannot predict on nil — XGBoost" if xs.nil?
 
         begin
+          @predicting = true
           y_pred = yield(preprocess(xs))
+          @predicting = false
+          y_pred
         rescue StandardError => e
           raise e unless e.message.match?(/Number of columns does not match/)
 
@@ -499,8 +502,10 @@ module EasyML
         feature_cols = exploded.columns
         features = lazy ? exploded.collect.to_numo : exploded.to_numo
 
-        weights = weights_col ? (lazy ? xs.select(weights_col).collect.to_numo : xs.select(weights_col).to_numo) : nil
-        weights = weights.flatten if weights
+        unless @predicting
+          weights = weights_col ? (lazy ? xs.select(weights_col).collect.to_numo : xs.select(weights_col).to_numo) : nil
+          weights = weights.flatten if weights
+        end
         if ys.present?
           ys = ys.is_a?(Array) ? Polars::Series.new(ys) : ys
           labels = lazy ? ys.collect.to_numo.flatten : ys.to_numo.flatten

data/app/models/easy_ml/prediction.rb

@@ -28,7 +28,7 @@ module EasyML
 
     def prediction
       prediction_value["value"]
-    end
+e   end
 
     def probabilities
       metadata["probabilities"]

data/app/models/easy_ml/splitters/base_splitter.rb

@@ -6,18 +6,14 @@ module EasyML
 
       attr_reader :splitter
 
-      def split(datasource, &block)
-        datasource.in_batches do |df|
-          split_df(df).tap do |splits|
-            yield splits if block_given?
-          end
-        end
-      end
-
       def split_df(df)
         df
       end
 
+      def split(dataset)
+        split_df(dataset.materialized_view)
+      end
+
       def initialize(splitter)
         @splitter = splitter
       end

data/app/models/easy_ml/splitters/date_splitter.rb

@@ -41,9 +41,10 @@ module EasyML
 
         validation_date_start, test_date_start = splits
 
+        dtype = df[date_col].dtype
         test_df = Polars.concat(
           [
-            df.filter(Polars.col(date_col) >= test_date_start),
+            df.filter(Polars.col(date_col).ge(Polars.lit(test_date_start).cast(dtype))),
             df.filter(Polars.col(date_col).is_null),
           ]
         )

data/app/models/easy_ml/splitters/predefined_splitter.rb

@@ -15,13 +15,18 @@ module EasyML
         }
       end
 
-      def split(datasource, &block)
+      def split(dataset, &block)
         validate!
 
-        files = datasource.all_files
+        files = dataset.datasource.all_files
         train, valid, test = match_files(files)
 
-        yield [reader.query(train), reader.query(valid), reader.query(test)]
+        values = [reader.query(train), reader.query(valid), reader.query(test)]
+        if block_given?
+          yield values
+        else
+          values
+        end
       end
 
       def match_files(files)

data/lib/easy_ml/data/dataset_manager/schema/normalizer.rb

@@ -0,0 +1,201 @@
+module EasyML
+  module Data
+    class DatasetManager
+      class Schema
+        class Normalizer
+
+          attr_accessor :files
+
+          def initialize(files)
+            @files = files
+          end
+
+          def normalize
+            shared_schema = find_common_schema(files)
+            if schema_changed?(files, shared_schema)
+              queries = schema_to_queries(shared_schema)
+              rewrite_dataset(files, queries)
+            end
+
+            queries = improve_schema(files, shared_schema)
+            if queries.any?
+              rewrite_dataset(files, queries)
+            end
+            files
+          end
+
+        private
+
+          def schema_changed?(files, schema)
+            Polars.scan_parquet(files.first).schema != schema
+          end
+
+          def rewrite_dataset(files, queries)
+            files.each do |file|
+              Polars.scan_parquet(file).select(queries).collect.write_parquet("#{file}_normalized.parquet")
+              puts "Rewriting #{file}..."
+              File.delete(file)
+              FileUtils.mv("#{file}_normalized.parquet", file)
+            end
+          end
+
+          def improve_schema(files, schema)
+            checks = schema_checks(schema)
+            return [] unless checks.any?
+
+            improvements = Polars.scan_parquet(files).select(checks).collect
+            conversions = improvements.to_hashes&.first || []
+            return [] unless conversions.any?
+            conversions = conversions&.select { |k,v| v }
+            return [] unless conversions.any?
+            
+            conversions = conversions.reduce({}) do |hash, (k, _)|
+              hash.tap do
+                key, ruby_type = k.split("convert_").last.split("_to_")
+                conversion = case ruby_type
+                            when "int"
+                              Polars.col(key).cast(Polars::Int64).alias(key)
+                            else
+                              EasyML::Data::DateConverter.conversion(k)
+                            end
+                hash[key] = conversion
+              end
+            end
+            schema.map do |k, v|
+              conversions[k] || Polars.col(k).cast(v).alias(k)
+            end
+          end
+
+          def schema_to_queries(schema)
+            schema.map do |k, v|
+              Polars.col(k).cast(v).alias(k)
+            end
+          end
+
+          def schema_checks(schema)
+            schema.flat_map do |key, value|
+              case value
+              when Polars::FloatType, Polars::Decimal
+                Polars.col(key).cast(Polars::Int64).cast(value).eq(Polars.col(key)).all().alias("convert_#{key}_to_int")
+              when Polars::String
+                EasyML::Data::DateConverter.queries(key)
+              end
+            end.compact
+          end
+
+          # Function to find a common schema across multiple parquet files
+          def find_common_schema(parquet_files)
+            # Get schema from each file
+            schemas = []
+            
+            parquet_files.each do |file|
+              begin
+                # Read just the schema without loading data
+                schema = Polars.scan_parquet(file).schema
+                schemas << schema
+              rescue => e
+                puts "Warning: Error reading schema from #{file}: #{e.message}"
+              end
+            end
+            
+            # Find common schema - start with first file's schema
+            return {} if schemas.empty?
+            
+            key_count = Hash.new(0)
+            common_schema = schemas.first
+            
+            # Reconcile types across all schemas
+            schemas.each do |schema|
+              schema.each do |name, dtype|
+                key_count[name] += 1
+                if common_schema.key?(name)
+                  # If types don't match, choose the more general type
+                  if common_schema[name] != dtype
+                    common_schema[name] = choose_compatible_type(common_schema[name], dtype)
+                  end
+                end
+              end
+            end
+            
+            # Filter out columns that aren't present in all files
+            common_schema = common_schema.select { |name, _| key_count[name] == schemas.length }
+            
+            return common_schema
+          end
+
+          # Choose a type that's compatible with both input types
+          def choose_compatible_type(type1, type2)
+            # Integer types - use the larger of the two
+            int_types = [Polars::Int8, Polars::Int16, Polars::Int32, Polars::Int64]
+            
+            # If both are integers, choose the larger one
+            if int_types.include?(type1.class) && int_types.include?(type2.class)
+              return [type1, type2].max_by { |t| int_types.index(t.class) }
+            end
+            
+            # If one is Int64 and one is Decimal with scale 0, use Decimal
+            if (type1.is_a?(Polars::Int64) && type2.is_a?(Polars::Decimal) && type2.scale == 0) ||
+              (type2.is_a?(Polars::Int64) && type1.is_a?(Polars::Decimal) && type1.scale == 0)
+              return type1.is_a?(Polars::Decimal) ? type1 : type2
+            end
+            
+            # If types are drastically different, convert to string as a safe fallback
+            if [Polars::String, Polars::Categorical].include?(type1.class) || 
+              [Polars::String, Polars::Categorical].include?(type2.class)
+              return Polars::String.new
+            end
+            
+            # For float vs decimal, choose decimal if it has scale > 0
+            if (type1.is_a?(Polars::Float64) && type2.is_a?(Polars::Decimal) && type2.scale > 0) ||
+              (type2.is_a?(Polars::Float64) && type1.is_a?(Polars::Decimal) && type1.scale > 0)
+              return type1.is_a?(Polars::Decimal) ? type1 : type2
+            end
+            
+            # Default to Float64 for numeric type conflicts
+            if [Polars::Float32, Polars::Float64, Polars::Decimal, Polars::Int64].any? { |t| type1.is_a?(t) } && 
+              [Polars::Float32, Polars::Float64, Polars::Decimal, Polars::Int64].any? { |t| type2.is_a?(t) }
+              return Polars::Float64.new
+            end
+            
+            # Fallback - use first type
+            return type1
+          end
+
+          # Apply a common schema to read all parquet files
+          def read_with_common_schema(parquet_files)
+            schema = find_common_schema(parquet_files)
+            return Polars.scan_parquet(parquet_files).with_schema(schema).collect
+          end
+
+          # Alternative approach using a union scan
+          def union_scan_parquet(parquet_files)
+            if parquet_files.empty?
+              return Polars.DataFrame.new
+            end
+            
+            # Create separate scans with explicit schemas
+            scans = []
+            schema = find_common_schema(parquet_files)
+            
+            parquet_files.each do |file|
+              scans << Polars.scan_parquet(file).with_schema(schema)
+            end
+            
+            # Union all scans
+            if scans.length == 1
+              return scans.first.collect
+            else
+              # Combine using concat (union all)
+              union = scans.first
+              scans[1..-1].each do |scan|
+                union = union.concat(scan)
+              end
+              
+              return union.collect
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/easy_ml/data/dataset_manager/schema.rb

@@ -0,0 +1,9 @@
+module EasyML
+  module Data
+    class DatasetManager
+      class Schema
+        require_relative "schema/normalizer"
+      end
+    end
+  end
+end