easy_ml 0.2.0.pre.rc57 → 0.2.0.pre.rc60

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e52412950fefc02e9b838930f132873c726440ebbc343159504d7d3287a39d05
-  data.tar.gz: 44ff18d1f1df78b542c8e536427189fce63d147e7e86623d219ed9b89c501ca7
+  metadata.gz: 43809758ea028bdeb30b9255c50fd951d7870cd5286d0e3ae9f0e30e09bf22a6
+  data.tar.gz: 5b2f0ae171a1043b8ce76dc438cab2931c2562f481de8c024beaeee8b15accfc
 SHA512:
-  metadata.gz: 1e543781fb426a6fa7fe6ad6f5b7c924bdab38d88ac8ad7288db3a24f683661b3745a6f2176c993899a9f9737af7e54dfa59cc439a71739d3e2d2d2d75714621
-  data.tar.gz: 3f012c5a3126eec7a69c3c11dd45017f7c2ded7a2bfd5e6e70bcaa388000b19e50d19ed15dc6b47786f61b698cc081e915abade7ece544a3c8a14d0a8f5c4696
+  metadata.gz: 012d1c40e4a4efdf330702effe53eef60b84f8ff7fb5277d0b0030192990f8e86ab9ec7e73ec80972328425c1bf53b8ccf9a275c0e083b7b8e13eeb04be5914b
+  data.tar.gz: b2b504db55932b4abcf8acd06ed19ea5eb9f3c79d246ac7a3398494f7220451e886ca35196c94816356542f774c11c12ca599595af2841aa76a14c0e6304e982

data/app/controllers/easy_ml/apis_controller.rb

@@ -0,0 +1,8 @@
+module EasyML
+  class APIsController < ApplicationController
+    def show
+      model = EasyML::Model.find_by!(slug: params[:model])
+      render json: { data: model.api_fields }
+    end
+  end
+end

data/app/controllers/easy_ml/application_controller.rb

@@ -23,6 +23,10 @@ module EasyML
       SettingsSerializer.new(settings).serializable_hash.dig(:data, :attributes)
     end
 
+    def dataset_to_json_small(dataset)
+      DatasetSerializer::SmallSerializer.new(dataset).serializable_hash.dig(:data, :attributes)
+    end
+
     def dataset_to_json(dataset)
       DatasetSerializer.new(dataset).serializable_hash.dig(:data, :attributes)
     end

data/app/controllers/easy_ml/datasets_controller.rb

@@ -26,7 +26,7 @@ module EasyML
       datasets = Dataset.all.order(id: :desc)
 
       render inertia: "pages/DatasetsPage", props: {
-        datasets: datasets.map { |dataset| dataset_to_json(dataset) },
+        datasets: datasets.map { |dataset| dataset_to_json_small(dataset) },
         constants: Dataset.constants,
       }
     end
@@ -76,6 +76,37 @@ module EasyML
         column_attrs[:preprocessing_steps] = nil if column_attrs.dig(:preprocessing_steps, :training, :method) == "none"
       end
 
+      # Handle feature ID assignment for existing features
+      if dataset_params[:features_attributes].present?
+        # Clean up any feature IDs that don't exist anymore
+        feature_ids = dataset_params[:features_attributes].map { |attrs| attrs[:id] }.compact
+        existing_feature_ids = Feature.where(id: feature_ids).pluck(:id)
+
+        params[:dataset][:features_attributes].each do |attrs|
+          if attrs[:id].present? && !existing_feature_ids.include?(attrs[:id].to_i)
+            attrs.delete(:id)
+          end
+        end
+
+        # Find existing features by feature_class
+        feature_classes = dataset_params[:features_attributes].map { |attrs|
+          attrs[:feature_class] if attrs[:id].blank?
+        }.compact
+
+        existing_features = Feature.where(feature_class: feature_classes)
+
+        # Update params with existing feature IDs
+        existing_features.each do |feature|
+          matching_param_index = params[:dataset][:features_attributes].find_index { |attrs|
+            attrs[:feature_class] == feature.feature_class
+          }
+
+          if matching_param_index
+            params[:dataset][:features_attributes][matching_param_index][:id] = feature.id
+          end
+        end
+      end
+
       if dataset.update(dataset_params)
         flash.now[:notice] = "Dataset configuration was successfully updated."
         render inertia: "pages/DatasetDetailsPage", props: {

data/app/controllers/easy_ml/models_controller.rb

@@ -53,6 +53,9 @@ module EasyML
         flash[:notice] = "Model was successfully created."
         redirect_to easy_ml_models_path
       else
+        errors = model.errors.to_hash(true)
+        values = errors.values.flatten
+        flash.now[:error] = values.join(", ")
         render inertia: "pages/NewModelPage", props: {
           datasets: EasyML::Dataset.all.map do |dataset|
             dataset.slice(:id, :name, :num_rows)

data/app/controllers/easy_ml/predictions_controller.rb

@@ -3,6 +3,11 @@ module EasyML
     skip_before_action :verify_authenticity_token, only: [:create]
 
     def create
+      slug = params[:model]
+      unless EasyML::Model.find_by(slug: slug).inference_version.present?
+        return render json: { error: "Model not found" }, status: :not_found
+      end
+
       unless params.key?(:input)
         return render json: { error: "Must provide key: input" }, status: :not_found
       end
@@ -12,17 +17,17 @@ module EasyML
         return render json: { error: "Input must be a hash" }, status: :not_found
       end
 
-      model_name = params[:model]
-      unless EasyML::Model.find_by(name: model_name).present?
-        return render json: { error: "Model not found" }, status: :not_found
+      valid, fields = EasyML::Predict.validate_input(slug, input)
+      unless valid
+        return render json: { error: "Missing required fields: #{fields}" }, status: :not_found
       end
 
-      prediction = EasyML::Predict.predict(model_name, input)
+      prediction = EasyML::Predict.predict(slug, input)
 
       render json: { prediction: EasyML::PredictionSerializer.new(prediction).serializable_hash.dig(:data, :attributes) }, status: :ok
     rescue ActiveRecord::RecordNotFound
       render json: { error: "Model not found" }, status: :not_found
-    rescue StandardError => e
+    rescue => e
       render json: { error: e.message }, status: :unprocessable_entity
     end
   end

data/app/frontend/components/DatasetPreview.tsx

@@ -58,9 +58,11 @@ export function DatasetPreview({ dataset }: DatasetPreviewProps) {
                   key={column.name}
                   className="bg-gray-50 rounded-lg p-4"
                 >
-                  <div className="flex items-center justify-between mb-2">
-                    <h4 className="font-medium text-gray-900">{column.name}</h4>
-                    <span className="text-xs font-medium text-gray-500 px-2 py-1 bg-gray-200 rounded-full">
+                  <div className="flex items-center justify-between mb-2 gap-2">
+                    <h4 className="font-medium text-gray-900 break-normal max-w-[70%] word-break:break-word overflow-wrap:anywhere whitespace-pre-wrap">
+                      {column.name.split('_').join('_\u200B')}
+                    </h4>
+                    <span className="text-xs font-medium text-gray-500 px-2 py-1 bg-gray-200 rounded-full flex-shrink-0">
                       {column.datatype}
                     </span>
                   </div>
@@ -68,23 +70,48 @@ export function DatasetPreview({ dataset }: DatasetPreviewProps) {
                   {column.statistics && (
                     <div className="space-y-1">
                       {Object.entries(column.statistics.raw).map(([key, value]) => {
-                        if (key === "counts") { 
+                        // Skip internal keys and null/undefined values
+                        if (key === "counts" || 
+                            key === "allowed_categories" ||
+                            key === "value" ||
+                            key === "label_encoder" ||
+                            key === "label_decoder" ||
+                            value === null || 
+                            value === undefined) { 
                           return null;
                         }
+
+                        // Format the value based on its type
+                        let displayValue: string;
+                        if (typeof value === 'number') {
+                          displayValue = value.toLocaleString(undefined, {
+                            maximumFractionDigits: 2
+                          });
+                        } else if (typeof value === 'object') {
+                          // Handle arrays or other objects
+                          displayValue = JSON.stringify(value);
+                        } else if (typeof value === 'boolean') {
+                          displayValue = value.toString();
+                        } else {
+                          displayValue = String(value);
+                        }
+
+                        // Truncate long strings
+                        if (displayValue.length > 50) {
+                          displayValue = displayValue.slice(0, 47) + '...';
+                        }
+
                         return (
-                        <div key={key} className="flex justify-between text-sm">
-                          <span className="text-gray-500">
-                            {key.charAt(0).toUpperCase() + key.slice(1)}:
-                          </span>
-                          <span className="font-medium text-gray-900">
-                            {typeof value === 'number' ? 
-                              value.toLocaleString(undefined, {
-                                maximumFractionDigits: 2
-                              }) : 
-                              value}
-                          </span>
-                        </div>
-                      )})}
+                          <div key={key} className="flex justify-between text-sm gap-2">
+                            <span className="text-gray-500 flex-shrink-0">
+                              {key.charAt(0).toUpperCase() + key.slice(1)}:
+                            </span>
+                            <span className="font-medium text-gray-900 text-right break-all">
+                              {displayValue}
+                            </span>
+                          </div>
+                        );
+                      })}
                     </div>
                   )}
                 </div>
@@ -142,10 +169,14 @@ export function DatasetPreview({ dataset }: DatasetPreviewProps) {
                     <tr key={i}>
                       {columns.map((column) => (
                         <td
-                          key={row[column]}
+                          key={`${i}-${column}`}
                           className="whitespace-nowrap px-3 py-4 text-sm text-gray-500"
                         >
-                          {row[column]?.toString()}
+                          {row[column] === null || row[column] === undefined 
+                            ? '' 
+                            : typeof row[column] === 'object' 
+                              ? JSON.stringify(row[column])
+                              : String(row[column])}
                         </td>
                       ))}
                     </tr>

data/app/frontend/components/ModelForm.tsx

@@ -74,7 +74,7 @@ export function ModelForm({ initialData, datasets, constants, isEditing, errors:
       dataset_id: initialData?.dataset_id || '',
       task: initialData?.task || 'classification',
       objective: initialData?.objective || 'binary:logistic',
-      metrics: initialData?.metrics || ['accuracy'],
+      metrics: initialData?.metrics || ['accuracy_score'],
       retraining_job_attributes: initialData?.retraining_job ? {
         id: initialData.retraining_job.id,
         frequency: initialData.retraining_job.frequency,

data/app/frontend/components/SearchableSelect.tsx

@@ -61,7 +61,6 @@ export const SearchableSelect = forwardRef<HTMLButtonElement, SearchableSelectPr
     }, [isOpen]);
 
     const handleOptionClick = (optionValue: Option['value'], e: React.MouseEvent) => {
-      debugger;
       e.preventDefault();
       e.stopPropagation();
       onChange(optionValue);

data/app/frontend/components/dataset/ColumnConfigModal.tsx

@@ -58,7 +58,7 @@ export function ColumnConfigModal({
   const [selectedColumn, setSelectedColumn] = useState<string | null>(null);
   const [searchQuery, setSearchQuery] = useState("");
   const [activeFilters, setActiveFilters] = useState<{
-    view: "all" | "training" | "hidden" | "preprocessed" | "nulls";
+    view: "all" | "training" | "hidden" | "preprocessed" | "nulls" | "computed" | "required";
     types: string[];
   }>({
     view: "all",
@@ -103,6 +103,10 @@ export function ColumnConfigModal({
             return colHasPreprocessingSteps(column);
           case "nulls":
             return (column.statistics?.processed?.null_count || 0) > 0;
+          case "computed":
+            return column.is_computed;
+          case "required":
+            return column.required;
           default:
             return true;
         }
@@ -124,6 +128,8 @@ export function ColumnConfigModal({
       withNulls: dataset.columns.filter(
         (c) => (c.statistics?.processed?.null_count || 0) > 0
       ).length,
+      computed: dataset.columns.filter((c) => c.is_computed === true).length,
+      required: dataset.columns.filter((c) => c.required === true).length,
     }),
     [dataset.columns, filteredColumns]
   );

data/app/frontend/components/dataset/ColumnFilters.tsx

@@ -1,16 +1,16 @@
 import React, { useState } from 'react';
-import { Filter, Database, Wrench, Eye, EyeOff, AlertTriangle, ChevronLeft, ChevronRight } from 'lucide-react';
+import { Filter, Database, Wrench, Eye, EyeOff, AlertTriangle, ChevronLeft, ChevronRight, Calculator, Target } from 'lucide-react';
 import type { Column } from '../../types';
 
 const ITEMS_PER_PAGE = 5;
 interface ColumnFiltersProps {
   types: string[];
   activeFilters: {
-    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls';
+    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls' | 'computed' | 'required';
     types: string[];
   };
   onFilterChange: (filters: {
-    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls';
+    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls' | 'computed' | 'required';
     types: string[];
   }) => void;
   columnStats: {
@@ -20,6 +20,8 @@ interface ColumnFiltersProps {
     hidden: number;
     withPreprocessing: number;
     withNulls: number;
+    computed: number;
+    required: number;
   };
   colHasPreprocessingSteps: (col: Column) => boolean;
   columns: Column[];
@@ -43,6 +45,10 @@ export function ColumnFilters({
         return `${columnStats.withPreprocessing} columns`;
       case 'nulls':
         return `${columnStats.withNulls} columns`;
+      case 'computed':
+        return `${columnStats.computed} columns`;
+      case 'required':
+        return `${columnStats.required} columns`;
       default:
         return `${columnStats.total} columns`;
     }
@@ -158,6 +164,34 @@ export function ColumnFilters({
               ({getViewStats('nulls')})
             </span>
           </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'computed' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'computed'
+                ? 'bg-purple-100 text-purple-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Calculator className="w-4 h-4" />
+            Computed
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('computed')})
+            </span>
+          </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'required' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'required'
+                ? 'bg-blue-100 text-blue-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Target className="w-4 h-4" />
+            Required
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('required')})
+            </span>
+          </button>
         </div>
 
         {/* Column Types */}

data/app/frontend/components/dataset/ColumnList.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Settings2, AlertCircle, Target, EyeOff, Eye } from 'lucide-react';
+import { Settings2, AlertCircle, Target, EyeOff, Eye, Calculator, Star } from 'lucide-react';
 import type { Column } from '../../types';
 import { usePage } from "@inertiajs/react";
 
@@ -79,11 +79,23 @@ export function ColumnList({
               </p>
             )}
             <div className="flex flex-wrap gap-2">
+              {column.required && (
+                <div className="flex items-center gap-1 text-blue-600">
+                  <Star className="w-3 h-3" />
+                  <span className="text-xs">required</span>
+                </div>
+              )}
+              {column.is_computed && (
+                <div className="flex items-center gap-1 text-purple-600">
+                  <Calculator className="w-3 h-3" />
+                  <span className="text-xs">computed</span>
+                </div>
+              )}
               {column.preprocessing_steps && column.preprocessing_steps?.training &&
                column.preprocessing_steps?.training?.method !== 'none' && (
                 <div className="flex items-center gap-1 text-blue-600">
                   <AlertCircle className="w-3 h-3" />
-                  <span className="text-xs">Preprocessing configured</span>
+                  <span className="text-xs">preprocessing configured</span>
                 </div>
               )}
               {column.hidden && (

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -1,5 +1,5 @@
 import React, { useState, useEffect } from 'react';
-import { Settings2, Wrench, ArrowRight, Pencil, Trash2, Database } from 'lucide-react';
+import { Settings2, Wrench, ArrowRight, Pencil, Trash2, Database, Calculator, GitBranch } from 'lucide-react';
 import type { Dataset, Column, ColumnType, PreprocessingConstants, PreprocessingSteps, PreprocessingStep } from '../../types/dataset';
 import { Badge } from "@/components/ui/badge";
 
@@ -250,7 +250,7 @@ export function PreprocessingConfig({
     setIsEditingDescription(true);
   };
 
-  let nullCount = (column.statistics?.processed.null_count || column.statistics?.raw.null_count) || 0;
+  let nullCount = (column.statistics?.processed.null_count || column.statistics?.raw?.null_count) || 0;
   const nullPercentage = nullCount && column.statistics?.raw.num_rows
     ? ((nullCount / column.statistics.raw.num_rows) * 100)
     : 0;
@@ -332,26 +332,41 @@ export function PreprocessingConfig({
             </div>
           </div>
           <div className="flex items-center gap-4 flex-shrink-0">
-            {column.is_target ? (
-              <span className="inline-flex items-center px-3 py-1 rounded-full text-sm font-medium bg-purple-100 text-purple-800">
-                Target Column
-              </span>
-            ) : (
-              <div className="flex items-center gap-2">
-                <label className="flex items-center gap-2 text-sm">
-                  <input
-                    type="checkbox"
-                    checked={column.drop_if_null}
-                    onChange={onToggleDropIfNull}
-                    className="rounded border-gray-300 text-red-600 focus:ring-red-500"
-                  />
-                  <span className="flex items-center gap-1 text-gray-700">
-                    <Trash2 className="w-4 h-4 text-gray-400" />
-                    Drop if null
-                  </span>
-                </label>
+            <div className="relative flex items-center gap-2">
+              <div className="absolute right-0 -top-8 flex items-center gap-2">
+                {column.required && (
+                  <Badge variant="secondary" className="bg-blue-100 text-blue-800">
+                    Required
+                  </Badge>
+                )}
+                {column.is_computed && (
+                  <Badge variant="secondary" className="bg-purple-100 text-purple-800">
+                    <Calculator className="w-3 h-3 mr-1" />
+                    Computed
+                  </Badge>
+                )}
               </div>
-            )}
+              {column.is_target ? (
+                <span className="inline-flex items-center px-3 py-1 rounded-full text-sm font-medium bg-purple-100 text-purple-800">
+                  Target Column
+                </span>
+              ) : (
+                <div className="flex items-center gap-2">
+                  <label className="flex items-center gap-2 text-sm">
+                    <input
+                      type="checkbox"
+                      checked={column.drop_if_null}
+                      onChange={onToggleDropIfNull}
+                      className="rounded border-gray-300 text-red-600 focus:ring-red-500"
+                    />
+                    <span className="flex items-center gap-1 text-gray-700">
+                      <Trash2 className="w-4 h-4 text-gray-400" />
+                      Drop if null
+                    </span>
+                  </label>
+                </div>
+              )}
+            </div>
           </div>
         </div>
 
@@ -481,6 +496,52 @@ export function PreprocessingConfig({
         )}
       </div>
 
+      {/* Column Lineage Section */}
+      {column.lineage && column.lineage.length > 0 && (
+        <div className="bg-white rounded-lg border border-gray-200 p-6">
+          <h3 className="text-lg font-medium text-gray-900 mb-4 flex items-center gap-2">
+            <GitBranch className="w-5 h-5 text-gray-500" />
+            Column Lineage
+          </h3>
+          <div className="space-y-4">
+            {column.lineage.map((step, index) => (
+              <div key={index} className="flex items-start gap-3">
+                <div className={`w-8 h-8 rounded-full flex items-center justify-center flex-shrink-0 ${
+                  step.key === 'raw_dataset' 
+                    ? 'bg-gray-100' 
+                    : step.key === 'computed_by_feature'
+                    ? 'bg-purple-100'
+                    : 'bg-blue-100'
+                }`}>
+                  {step.key === 'raw_dataset' ? (
+                    <Database className="w-4 h-4 text-gray-600" />
+                  ) : step.key === 'computed_by_feature' ? (
+                    <Calculator className="w-4 h-4 text-purple-600" />
+                  ) : (
+                    <Settings2 className="w-4 h-4 text-blue-600" />
+                  )}
+                </div>
+                <div className="flex-1">
+                  <div className="flex items-center justify-between">
+                    <p className="text-sm font-medium text-gray-900">
+                      {step.description}
+                    </p>
+                    {step.timestamp && (
+                      <span className="text-xs text-gray-500">
+                        {new Date(step.timestamp).toLocaleString()}
+                      </span>
+                    )}
+                  </div>
+                  {index < column.lineage.length - 1 && (
+                    <div className="ml-4 mt-2 mb-2 w-0.5 h-4 bg-gray-200" />
+                  )}
+                </div>
+              </div>
+            ))}
+          </div>
+        </div>
+      )}
+
       {/* Data Type Section */}
       <div className="bg-white rounded-lg border border-gray-200 p-6">
         <h3 className="text-lg font-medium text-gray-900 mb-4 flex items-center gap-2">

data/app/frontend/pages/DatasourcesPage.tsx

@@ -49,12 +49,10 @@ export default function DatasourcesPage({ datasources }: { datasources: Datasour
         preserveScroll: true, // Keeps the scroll position
         preserveState: true,  // Keeps the form state
         onSuccess: (e) => {
-          debugger;
           console.log("SUCCESS")
           // The page will automatically refresh with new data
         },
         onError: () => {
-          debugger;
           // Handle error case if needed
           console.error('Failed to sync datasource');
         }

data/app/frontend/types/dataset.ts

@@ -84,6 +84,9 @@ export interface Column {
   sample_values: {};
   statistics?: Statistics;
   preprocessing_steps?: PreprocessingSteps;
+  lineage?: Array<{ key: string }>;
+  required?: boolean;
+  is_computed?: boolean;
 }
 
 export interface Dataset {

data/app/jobs/easy_ml/compute_feature_job.rb

@@ -13,7 +13,6 @@ module EasyML
 
         # Check if any feature has failed before proceeding
         if dataset.features.any? { |f| f.workflow_status == "failed" }
-          puts "Aborting feature computation due to previous feature failure"
           return
         end
 
@@ -39,7 +38,6 @@ module EasyML
     end
 
     def self.after_batch_hook(batch_id, *args)
-      puts "After batch!"
       batch_args = fetch_batch_arguments(batch_id).flatten.map(&:symbolize_keys)
       feature_ids = batch_args.pluck(:feature_id).uniq
       parent_id = batch_args.pluck(:parent_batch_id).first

data/app/jobs/easy_ml/refresh_dataset_job.rb

@@ -3,28 +3,22 @@ module EasyML
     def perform(id)
       begin
         dataset = EasyML::Dataset.find(id)
-        return if dataset.workflow_status == :analyzing
 
         puts "Refreshing dataset #{dataset.name}"
-        puts "Needs refresh? #{dataset.needs_refresh?}"
         unless dataset.needs_refresh?
           dataset.update(workflow_status: :ready)
         end
 
         create_event(dataset, "started")
 
-        puts "Prepare! #{dataset.name}"
         dataset.unlock!
         dataset.prepare
         if dataset.features.needs_fit.any?
           dataset.fit_features(async: true)
-          puts "Computing features!"
         else
           dataset.actually_refresh
-          puts "Done!"
         end
       rescue StandardError => e
-        puts "Error #{e.message}"
         if Rails.env.test?
           raise e
         end