easy_ml 0.2.0.pre.rc58 → 0.2.0.pre.rc61

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a25c50b89c079e7e62f52d1f5a52ef16f3d7bc9b388fcee9a7b0983148de9cd
-  data.tar.gz: bfcf0d06fbe498ccc70251c144649d7fd6699c2bdc4a9acbf5866e60ce04c7bd
+  metadata.gz: 1b194e446191bb71affc096c0dbc049f2b0dc76878b83d4ff1eb729e9d8a29b9
+  data.tar.gz: c33ef37f01db6b965433c8fe90e487905e17ebdfcf6967fa93ef6cf10c8755eb
 SHA512:
-  metadata.gz: 77186b1d2d7558db7d128e03c68f8632af6f28be4b1bf2daa71ac804abbcc5a26470fe29a802258d326048155fea46a7f707e46fc88cf8571af5f30cb870d839
-  data.tar.gz: a22ba3e21ab32e64674033f0c83d023e41c3c5f158117be5fa8b85f1865a4bfc39bda72042e4c1ff80277d4d664fcc694349f7a377933c7c52f23417c299618b
+  metadata.gz: 92d0d03c4d258942c01967e6afe1ec2dc956a2a70b5d95d400f44189948b4192c9cd5a599c8d5b48cae2c0b72be18ee7926dfb6b4555d32e3e6292e03070f30e
+  data.tar.gz: 57b3e65c77f765ecce66fdcfaa0bce24c2c31a1b5f9b248a25dcd2a5fc587450df3dee496f9cd75a6c7754c6cc5cef173c04c9c04d8da495f0dbfbb95b97d6e1

data/app/controllers/easy_ml/application_controller.rb

@@ -23,6 +23,10 @@ module EasyML
       SettingsSerializer.new(settings).serializable_hash.dig(:data, :attributes)
     end
 
+    def dataset_to_json_small(dataset)
+      DatasetSerializer::SmallSerializer.new(dataset).serializable_hash.dig(:data, :attributes)
+    end
+
     def dataset_to_json(dataset)
       DatasetSerializer.new(dataset).serializable_hash.dig(:data, :attributes)
     end

data/app/controllers/easy_ml/datasets_controller.rb

@@ -26,7 +26,7 @@ module EasyML
       datasets = Dataset.all.order(id: :desc)
 
       render inertia: "pages/DatasetsPage", props: {
-        datasets: datasets.map { |dataset| dataset_to_json(dataset) },
+        datasets: datasets.map { |dataset| dataset_to_json_small(dataset) },
         constants: Dataset.constants,
       }
     end
@@ -76,6 +76,37 @@ module EasyML
         column_attrs[:preprocessing_steps] = nil if column_attrs.dig(:preprocessing_steps, :training, :method) == "none"
       end
 
+      # Handle feature ID assignment for existing features
+      if dataset_params[:features_attributes].present?
+        # Clean up any feature IDs that don't exist anymore
+        feature_ids = dataset_params[:features_attributes].map { |attrs| attrs[:id] }.compact
+        existing_feature_ids = Feature.where(id: feature_ids).pluck(:id)
+
+        params[:dataset][:features_attributes].each do |attrs|
+          if attrs[:id].present? && !existing_feature_ids.include?(attrs[:id].to_i)
+            attrs.delete(:id)
+          end
+        end
+
+        # Find existing features by feature_class
+        feature_classes = dataset_params[:features_attributes].map { |attrs|
+          attrs[:feature_class] if attrs[:id].blank?
+        }.compact
+
+        existing_features = Feature.where(feature_class: feature_classes)
+
+        # Update params with existing feature IDs
+        existing_features.each do |feature|
+          matching_param_index = params[:dataset][:features_attributes].find_index { |attrs|
+            attrs[:feature_class] == feature.feature_class
+          }
+
+          if matching_param_index
+            params[:dataset][:features_attributes][matching_param_index][:id] = feature.id
+          end
+        end
+      end
+
       if dataset.update(dataset_params)
         flash.now[:notice] = "Dataset configuration was successfully updated."
         render inertia: "pages/DatasetDetailsPage", props: {

data/app/frontend/components/DatasetPreview.tsx

@@ -58,9 +58,11 @@ export function DatasetPreview({ dataset }: DatasetPreviewProps) {
                   key={column.name}
                   className="bg-gray-50 rounded-lg p-4"
                 >
-                  <div className="flex items-center justify-between mb-2">
-                    <h4 className="font-medium text-gray-900">{column.name}</h4>
-                    <span className="text-xs font-medium text-gray-500 px-2 py-1 bg-gray-200 rounded-full">
+                  <div className="flex items-center justify-between mb-2 gap-2">
+                    <h4 className="font-medium text-gray-900 break-normal max-w-[70%] word-break:break-word overflow-wrap:anywhere whitespace-pre-wrap">
+                      {column.name.split('_').join('_\u200B')}
+                    </h4>
+                    <span className="text-xs font-medium text-gray-500 px-2 py-1 bg-gray-200 rounded-full flex-shrink-0">
                       {column.datatype}
                     </span>
                   </div>
@@ -68,23 +70,48 @@ export function DatasetPreview({ dataset }: DatasetPreviewProps) {
                   {column.statistics && (
                     <div className="space-y-1">
                       {Object.entries(column.statistics.raw).map(([key, value]) => {
-                        if (key === "counts") { 
+                        // Skip internal keys and null/undefined values
+                        if (key === "counts" || 
+                            key === "allowed_categories" ||
+                            key === "value" ||
+                            key === "label_encoder" ||
+                            key === "label_decoder" ||
+                            value === null || 
+                            value === undefined) { 
                           return null;
                         }
+
+                        // Format the value based on its type
+                        let displayValue: string;
+                        if (typeof value === 'number') {
+                          displayValue = value.toLocaleString(undefined, {
+                            maximumFractionDigits: 2
+                          });
+                        } else if (typeof value === 'object') {
+                          // Handle arrays or other objects
+                          displayValue = JSON.stringify(value);
+                        } else if (typeof value === 'boolean') {
+                          displayValue = value.toString();
+                        } else {
+                          displayValue = String(value);
+                        }
+
+                        // Truncate long strings
+                        if (displayValue.length > 50) {
+                          displayValue = displayValue.slice(0, 47) + '...';
+                        }
+
                         return (
-                        <div key={key} className="flex justify-between text-sm">
-                          <span className="text-gray-500">
-                            {key.charAt(0).toUpperCase() + key.slice(1)}:
-                          </span>
-                          <span className="font-medium text-gray-900">
-                            {typeof value === 'number' ? 
-                              value.toLocaleString(undefined, {
-                                maximumFractionDigits: 2
-                              }) : 
-                              value}
-                          </span>
-                        </div>
-                      )})}
+                          <div key={key} className="flex justify-between text-sm gap-2">
+                            <span className="text-gray-500 flex-shrink-0">
+                              {key.charAt(0).toUpperCase() + key.slice(1)}:
+                            </span>
+                            <span className="font-medium text-gray-900 text-right break-all">
+                              {displayValue}
+                            </span>
+                          </div>
+                        );
+                      })}
                     </div>
                   )}
                 </div>
@@ -142,10 +169,14 @@ export function DatasetPreview({ dataset }: DatasetPreviewProps) {
                     <tr key={i}>
                       {columns.map((column) => (
                         <td
-                          key={row[column]}
+                          key={`${i}-${column}`}
                           className="whitespace-nowrap px-3 py-4 text-sm text-gray-500"
                         >
-                          {row[column]?.toString()}
+                          {row[column] === null || row[column] === undefined 
+                            ? '' 
+                            : typeof row[column] === 'object' 
+                              ? JSON.stringify(row[column])
+                              : String(row[column])}
                         </td>
                       ))}
                     </tr>

data/app/frontend/components/dataset/ColumnConfigModal.tsx

@@ -58,7 +58,7 @@ export function ColumnConfigModal({
   const [selectedColumn, setSelectedColumn] = useState<string | null>(null);
   const [searchQuery, setSearchQuery] = useState("");
   const [activeFilters, setActiveFilters] = useState<{
-    view: "all" | "training" | "hidden" | "preprocessed" | "nulls";
+    view: "all" | "training" | "hidden" | "preprocessed" | "nulls" | "computed" | "required";
     types: string[];
   }>({
     view: "all",
@@ -103,6 +103,10 @@ export function ColumnConfigModal({
             return colHasPreprocessingSteps(column);
           case "nulls":
             return (column.statistics?.processed?.null_count || 0) > 0;
+          case "computed":
+            return column.is_computed;
+          case "required":
+            return column.required;
           default:
             return true;
         }
@@ -124,6 +128,8 @@ export function ColumnConfigModal({
       withNulls: dataset.columns.filter(
         (c) => (c.statistics?.processed?.null_count || 0) > 0
       ).length,
+      computed: dataset.columns.filter((c) => c.is_computed === true).length,
+      required: dataset.columns.filter((c) => c.required === true).length,
     }),
     [dataset.columns, filteredColumns]
   );

data/app/frontend/components/dataset/ColumnFilters.tsx

@@ -1,16 +1,16 @@
 import React, { useState } from 'react';
-import { Filter, Database, Wrench, Eye, EyeOff, AlertTriangle, ChevronLeft, ChevronRight } from 'lucide-react';
+import { Filter, Database, Wrench, Eye, EyeOff, AlertTriangle, ChevronLeft, ChevronRight, Calculator, Target } from 'lucide-react';
 import type { Column } from '../../types';
 
 const ITEMS_PER_PAGE = 5;
 interface ColumnFiltersProps {
   types: string[];
   activeFilters: {
-    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls';
+    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls' | 'computed' | 'required';
     types: string[];
   };
   onFilterChange: (filters: {
-    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls';
+    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls' | 'computed' | 'required';
     types: string[];
   }) => void;
   columnStats: {
@@ -20,6 +20,8 @@ interface ColumnFiltersProps {
     hidden: number;
     withPreprocessing: number;
     withNulls: number;
+    computed: number;
+    required: number;
   };
   colHasPreprocessingSteps: (col: Column) => boolean;
   columns: Column[];
@@ -43,6 +45,10 @@ export function ColumnFilters({
         return `${columnStats.withPreprocessing} columns`;
       case 'nulls':
         return `${columnStats.withNulls} columns`;
+      case 'computed':
+        return `${columnStats.computed} columns`;
+      case 'required':
+        return `${columnStats.required} columns`;
       default:
         return `${columnStats.total} columns`;
     }
@@ -158,6 +164,34 @@ export function ColumnFilters({
               ({getViewStats('nulls')})
             </span>
           </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'computed' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'computed'
+                ? 'bg-purple-100 text-purple-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Calculator className="w-4 h-4" />
+            Computed
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('computed')})
+            </span>
+          </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'required' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'required'
+                ? 'bg-blue-100 text-blue-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Target className="w-4 h-4" />
+            Required
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('required')})
+            </span>
+          </button>
         </div>
 
         {/* Column Types */}

data/app/frontend/components/dataset/ColumnList.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Settings2, AlertCircle, Target, EyeOff, Eye } from 'lucide-react';
+import { Settings2, AlertCircle, Target, EyeOff, Eye, Calculator, Star } from 'lucide-react';
 import type { Column } from '../../types';
 import { usePage } from "@inertiajs/react";
 
@@ -79,11 +79,23 @@ export function ColumnList({
               </p>
             )}
             <div className="flex flex-wrap gap-2">
+              {column.required && (
+                <div className="flex items-center gap-1 text-blue-600">
+                  <Star className="w-3 h-3" />
+                  <span className="text-xs">required</span>
+                </div>
+              )}
+              {column.is_computed && (
+                <div className="flex items-center gap-1 text-purple-600">
+                  <Calculator className="w-3 h-3" />
+                  <span className="text-xs">computed</span>
+                </div>
+              )}
               {column.preprocessing_steps && column.preprocessing_steps?.training &&
                column.preprocessing_steps?.training?.method !== 'none' && (
                 <div className="flex items-center gap-1 text-blue-600">
                   <AlertCircle className="w-3 h-3" />
-                  <span className="text-xs">Preprocessing configured</span>
+                  <span className="text-xs">preprocessing configured</span>
                 </div>
               )}
               {column.hidden && (

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -1,5 +1,5 @@
 import React, { useState, useEffect } from 'react';
-import { Settings2, Wrench, ArrowRight, Pencil, Trash2, Database } from 'lucide-react';
+import { Settings2, Wrench, ArrowRight, Pencil, Trash2, Database, Calculator, GitBranch } from 'lucide-react';
 import type { Dataset, Column, ColumnType, PreprocessingConstants, PreprocessingSteps, PreprocessingStep } from '../../types/dataset';
 import { Badge } from "@/components/ui/badge";
 
@@ -332,26 +332,41 @@ export function PreprocessingConfig({
             </div>
           </div>
           <div className="flex items-center gap-4 flex-shrink-0">
-            {column.is_target ? (
-              <span className="inline-flex items-center px-3 py-1 rounded-full text-sm font-medium bg-purple-100 text-purple-800">
-                Target Column
-              </span>
-            ) : (
-              <div className="flex items-center gap-2">
-                <label className="flex items-center gap-2 text-sm">
-                  <input
-                    type="checkbox"
-                    checked={column.drop_if_null}
-                    onChange={onToggleDropIfNull}
-                    className="rounded border-gray-300 text-red-600 focus:ring-red-500"
-                  />
-                  <span className="flex items-center gap-1 text-gray-700">
-                    <Trash2 className="w-4 h-4 text-gray-400" />
-                    Drop if null
-                  </span>
-                </label>
+            <div className="relative flex items-center gap-2">
+              <div className="absolute right-0 -top-8 flex items-center gap-2">
+                {column.required && (
+                  <Badge variant="secondary" className="bg-blue-100 text-blue-800">
+                    Required
+                  </Badge>
+                )}
+                {column.is_computed && (
+                  <Badge variant="secondary" className="bg-purple-100 text-purple-800">
+                    <Calculator className="w-3 h-3 mr-1" />
+                    Computed
+                  </Badge>
+                )}
               </div>
-            )}
+              {column.is_target ? (
+                <span className="inline-flex items-center px-3 py-1 rounded-full text-sm font-medium bg-purple-100 text-purple-800">
+                  Target Column
+                </span>
+              ) : (
+                <div className="flex items-center gap-2">
+                  <label className="flex items-center gap-2 text-sm">
+                    <input
+                      type="checkbox"
+                      checked={column.drop_if_null}
+                      onChange={onToggleDropIfNull}
+                      className="rounded border-gray-300 text-red-600 focus:ring-red-500"
+                    />
+                    <span className="flex items-center gap-1 text-gray-700">
+                      <Trash2 className="w-4 h-4 text-gray-400" />
+                      Drop if null
+                    </span>
+                  </label>
+                </div>
+              )}
+            </div>
           </div>
         </div>
 
@@ -481,6 +496,52 @@ export function PreprocessingConfig({
         )}
       </div>
 
+      {/* Column Lineage Section */}
+      {column.lineage && column.lineage.length > 0 && (
+        <div className="bg-white rounded-lg border border-gray-200 p-6">
+          <h3 className="text-lg font-medium text-gray-900 mb-4 flex items-center gap-2">
+            <GitBranch className="w-5 h-5 text-gray-500" />
+            Column Lineage
+          </h3>
+          <div className="space-y-4">
+            {column.lineage.map((step, index) => (
+              <div key={index} className="flex items-start gap-3">
+                <div className={`w-8 h-8 rounded-full flex items-center justify-center flex-shrink-0 ${
+                  step.key === 'raw_dataset' 
+                    ? 'bg-gray-100' 
+                    : step.key === 'computed_by_feature'
+                    ? 'bg-purple-100'
+                    : 'bg-blue-100'
+                }`}>
+                  {step.key === 'raw_dataset' ? (
+                    <Database className="w-4 h-4 text-gray-600" />
+                  ) : step.key === 'computed_by_feature' ? (
+                    <Calculator className="w-4 h-4 text-purple-600" />
+                  ) : (
+                    <Settings2 className="w-4 h-4 text-blue-600" />
+                  )}
+                </div>
+                <div className="flex-1">
+                  <div className="flex items-center justify-between">
+                    <p className="text-sm font-medium text-gray-900">
+                      {step.description}
+                    </p>
+                    {step.timestamp && (
+                      <span className="text-xs text-gray-500">
+                        {new Date(step.timestamp).toLocaleString()}
+                      </span>
+                    )}
+                  </div>
+                  {index < column.lineage.length - 1 && (
+                    <div className="ml-4 mt-2 mb-2 w-0.5 h-4 bg-gray-200" />
+                  )}
+                </div>
+              </div>
+            ))}
+          </div>
+        </div>
+      )}
+
       {/* Data Type Section */}
       <div className="bg-white rounded-lg border border-gray-200 p-6">
         <h3 className="text-lg font-medium text-gray-900 mb-4 flex items-center gap-2">

data/app/frontend/types/dataset.ts

@@ -84,6 +84,9 @@ export interface Column {
   sample_values: {};
   statistics?: Statistics;
   preprocessing_steps?: PreprocessingSteps;
+  lineage?: Array<{ key: string }>;
+  required?: boolean;
+  is_computed?: boolean;
 }
 
 export interface Dataset {

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -5,7 +5,6 @@ module EasyML
     @queue = :easy_ml
 
     def self.perform(batch_id, options = {})
-      puts "Performing compute feature job with options #{options}"
       begin
         options.symbolize_keys!
         feature_id = options.dig(:feature_id)
@@ -14,7 +13,6 @@ module EasyML
 
         # Check if any feature has failed before proceeding
         if dataset.features.any? { |f| f.workflow_status == "failed" }
-          puts "Aborting feature computation due to previous feature failure"
           return
         end
 
@@ -40,7 +38,6 @@ module EasyML
     end
 
     def self.after_batch_hook(batch_id, *args)
-      puts "After batch!"
       batch_args = fetch_batch_arguments(batch_id).flatten.map(&:symbolize_keys)
       feature_ids = batch_args.pluck(:feature_id).uniq
       parent_id = batch_args.pluck(:parent_batch_id).first

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -3,28 +3,22 @@ module EasyML
     def perform(id)
       begin
         dataset = EasyML::Dataset.find(id)
-        return if dataset.workflow_status == :analyzing
 
         puts "Refreshing dataset #{dataset.name}"
-        puts "Needs refresh? #{dataset.needs_refresh?}"
         unless dataset.needs_refresh?
           dataset.update(workflow_status: :ready)
         end
 
         create_event(dataset, "started")
 
-        puts "Prepare! #{dataset.name}"
         dataset.unlock!
         dataset.prepare
         if dataset.features.needs_fit.any?
           dataset.fit_features(async: true)
-          puts "Computing features!"
         else
           dataset.actually_refresh
-          puts "Done!"
         end
       rescue StandardError => e
-        puts "Error #{e.message}"
         if Rails.env.test?
           raise e
         end

data/app/models/easy_ml/column/imputers/base.rb

@@ -0,0 +1,89 @@
+module EasyML
+  class Column
+    class Imputers
+      class Base
+        class << self
+          def param_applies(p)
+            Imputers.supported_params << p
+            Imputers.params_by_class[self] ||= []
+            Imputers.params_by_class[self] << p.to_sym
+          end
+
+          def method_applies(m)
+            Imputers.supported_methods << m.to_sym
+            Imputers.methods_by_class[self] ||= []
+            Imputers.methods_by_class[self] << m.to_sym
+          end
+
+          def description
+            "Unknown preprocessing method"
+          end
+        end
+
+        attr_accessor :column, :preprocessing_step
+
+        def initialize(column, preprocessing_step)
+          @column = column
+          @preprocessing_step = preprocessing_step.with_indifferent_access
+        end
+
+        def applies?
+          method_applies? || param_applies?
+        end
+
+        def method_applies?
+          imputers_own_methods.include?(method.to_sym)
+        end
+
+        def param_applies?
+          params.keys.any? { |p| imputers_own_params.include?(p.to_sym) && params[p] != false }
+        end
+
+        def imputers_own_methods
+          Imputers.methods_by_class[self.class] || []
+        end
+
+        def imputers_own_params
+          Imputers.params_by_class[self.class] || []
+        end
+
+        def params
+          @preprocessing_step.dig(:params)
+        end
+
+        def method
+          @preprocessing_step.dig(:method)
+        end
+
+        def statistics(*args)
+          if column.is_computed
+            column.statistics.dig(:processed, *args)
+          else
+            column.statistics.dig(:clipped, *args) || column.statistics.dig(:raw, *args)
+          end
+        end
+
+        def anything?
+          true
+        end
+
+        def inspect
+          params_str = params ? params.map { |k, v| "#{k}: #{v}" }.join(", ") : "none"
+          method_str = method ? method : "none"
+
+          "#<#{self.class.name} method=#{method_str.inspect} params={#{params_str}}>"
+        end
+
+        alias_method :to_s, :inspect
+
+        def transform(df)
+          raise "Method not implemented"
+        end
+
+        def description
+          self.class.description
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/categorical.rb

@@ -0,0 +1,35 @@
+module EasyML
+  class Column
+    class Imputers
+      class Categorical < Base
+        method_applies :categorical
+        param_applies :categorical_min
+
+        def self.description
+          "Categorical imputation"
+        end
+
+        def transform(df)
+          return df unless allowed_categories.present?
+
+          case column.datatype
+          when :categorical
+            df = df.with_column(
+              Polars.when(Polars.col(column.name).is_in(allowed_categories))
+                .then(Polars.col(column.name))
+                .otherwise(Polars.lit("other"))
+                .alias(column.name)
+            )
+          when :boolean
+            # no-op
+          end
+          df
+        end
+
+        def allowed_categories
+          column.allowed_categories
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/clip.rb

@@ -0,0 +1,30 @@
+module EasyML
+  class Column
+    class Imputers
+      class Clip < Base
+        attr_accessor :column, :dataset, :preprocessing_step
+
+        param_applies :clip
+
+        def self.description
+          "Clip"
+        end
+
+        def transform(df)
+          df = df.with_column(
+            Polars.col(column.name).clip(min, max).alias(column.name)
+          )
+          df
+        end
+
+        def min
+          params.dig(:clip, :min) || 0
+        end
+
+        def max
+          params.dig(:clip, :max) || Float::INFINITY
+        end
+      end
+    end
+  end
+end

data/app/models/easy_ml/column/imputers/constant.rb

@@ -0,0 +1,27 @@
+module EasyML
+  class Column
+    class Imputers
+      class Constant < Base
+        method_applies :constant
+        param_applies :constant
+
+        def self.description
+          "Constant value imputation"
+        end
+
+        def transform(df)
+          return df unless constant.present?
+
+          df = df.with_column(
+            Polars.col(column.name).fill_null(constant).alias(column.name)
+          )
+          df
+        end
+
+        def constant
+          params.dig(:constant)
+        end
+      end
+    end
+  end
+end