easy_ml 0.2.0.pre.rc45 → 0.2.0.pre.rc46

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6df30d23d49530894614483d7bebe0599aa1aa1fc62108024b7c417cd1bb550
-  data.tar.gz: ce317fa72227d618313d9b41b1d632d18628029253d33a69834c7ca878f18c4c
+  metadata.gz: 6b4c7d737a4d6775aa7bc7be7408994b4a68e76f55288563ed56990fe1aa2bb2
+  data.tar.gz: d1c147bbc6087489e908cb2df4f61de0021f553d4673f82955929177b4ddd754
 SHA512:
-  metadata.gz: 1508d885e06d23f8e255d56499e2ab6757e9a5725b1f367a66b2b0a6e904c15aff437f0dc3b660a067e2d45730659248af4ff31f0dc02198b840735c3845c7a2
-  data.tar.gz: 3631f30b7825a0cb5a6a5b6a0044fbfc5ce2bc4f627d395c66045f34c9c2fde43be249e996ee37ab9eb36ef5bd74cbfdfcc4ad6d5f93325ea07a48914f29efd4
+  metadata.gz: 9f182c7d0d9cd5f01124d4a3af4cb9c2f99f1450e3f98e77151761b77fd387cd8ee810d38c9797a8af182f44bc6896ab7789d71f2d475d962733fbd8cd8f0e5a
+  data.tar.gz: 1d4d930f6be09ab56838e79ae9b52a464ebbc25905644557aa2911bd1d2dfdffe96bc7563753b02a9d09c1016da050ef7c6e2ddf2124f9b11d1cddc4775f4792

data/app/controllers/easy_ml/datasets_controller.rb

@@ -23,7 +23,7 @@
 module EasyML
   class DatasetsController < ApplicationController
     def index
-      datasets = Dataset.all
+      datasets = Dataset.all.order(id: :desc)
 
       render inertia: "pages/DatasetsPage", props: {
         datasets: datasets.map { |dataset| dataset_to_json(dataset) },
@@ -131,6 +131,7 @@ module EasyML
           :datatype,
           :polars_datatype,
           :is_target,
+          :is_date_column,
           :hidden,
           :drop_if_null,
           :sample_values,

data/app/frontend/components/dataset/ColumnConfigModal.tsx

@@ -10,6 +10,7 @@ import {
   Play,
   Loader2,
   Sparkles,
+  Calendar,
 } from "lucide-react";
 import { PreprocessingConfig } from "./PreprocessingConfig";
 import { ColumnList } from "./ColumnList";
@@ -24,6 +25,7 @@ import { router } from "@inertiajs/react";
 
 interface ColumnConfig {
   targetColumn?: string;
+  dateColumn?: string;
 }
 
 interface ColumnConfigModalProps {
@@ -45,9 +47,13 @@ export function ColumnConfigModal({
   const [activeTab, setActiveTab] = useState<"columns" | "features">(
     "columns"
   );
+  const [activeColumnSubTab, setActiveColumnSubTab] = useState<
+    "target" | "date"
+  >("target");
   const [isApplying, setIsApplying] = useState(false);
   const [config, setConfig] = useState<ColumnConfig>({
     targetColumn: dataset.target,
+    dateColumn: dataset.date_column,
   });
   const [selectedColumn, setSelectedColumn] = useState<string | null>(null);
   const [searchQuery, setSearchQuery] = useState("");
@@ -127,6 +133,15 @@ export function ColumnConfigModal({
     [dataset.columns]
   );
 
+  const dateColumnOptions = useMemo(() => {
+    return dataset.columns
+      .filter((column) => column.datatype === "datetime")
+      .map((column) => ({
+        value: column.name,
+        label: column.name,
+      }));
+  }, [dataset.columns]);
+
   const handleColumnSelect = (columnName: string) => {
     setSelectedColumn(columnName);
   };
@@ -146,7 +161,7 @@ export function ColumnConfigModal({
 
   const setTargetColumn = (columnName: string) => {
     const name = String(columnName);
-    setConfig({ targetColumn: columnName });
+    setConfig({ ...config, targetColumn: columnName });
     const updatedColumns = dataset.columns.map((c) => ({
       ...c,
       is_target: c.name === name,
@@ -159,6 +174,21 @@ export function ColumnConfigModal({
     setNeedsRefresh(true);
   };
 
+  const setDateColumn = (columnName: string) => {
+    const name = String(columnName);
+    setConfig((prev) => ({ ...prev, dateColumn: columnName }));
+    const updatedColumns = dataset.columns.map((c) => ({
+      ...c,
+      is_date_column: c.name === name,
+    }));
+
+    setDataset({
+      ...dataset,
+      columns: updatedColumns,
+    });
+    setNeedsRefresh(true);
+  };
+
   const setColumnType = (columnName: string, datatype: string) => {
     const updatedColumns = dataset.columns.map((c) => ({
       ...c,
@@ -285,32 +315,25 @@ export function ColumnConfigModal({
         <div className="flex border-b shrink-0">
           <button
             onClick={() => setActiveTab("columns")}
-            className={`px-4 py-2 text-sm font-medium border-b-2 ${
+            className={`flex items-center gap-2 px-4 py-2 border-b-2 ${
               activeTab === "columns"
                 ? "border-blue-500 text-blue-600"
                 : "border-transparent text-gray-500 hover:text-gray-700"
             }`}
           >
-            <div className="flex items-center gap-2">
-              <Settings2 className="w-4 h-4" />
-              Column Configuration
-            </div>
+            <Settings2 className="w-4 h-4" />
+            Preprocessing
           </button>
           <button
             onClick={() => setActiveTab("features")}
-            className={`px-4 py-2 text-sm font-medium border-b-2 ${
+            className={`flex items-center gap-2 px-4 py-2 border-b-2 ${
               activeTab === "features"
                 ? "border-blue-500 text-blue-600"
                 : "border-transparent text-gray-500 hover:text-gray-700"
             }`}
           >
-            <div className="flex items-center gap-2">
-              <Wand2 className="w-4 h-4" />
-              Features
-              <span className="px-1.5 py-0.5 text-xs font-medium bg-blue-100 text-blue-600 rounded-full">
-                {constants.feature_options.length}
-              </span>
-            </div>
+            <Wand2 className="w-4 h-4" />
+            Feature Engineering
           </button>
 
           {needsRefresh && (
@@ -341,20 +364,60 @@ export function ColumnConfigModal({
           <React.Fragment>
             <div className="grid grid-cols-7 flex-1 min-h-0">
               <div className="col-span-3 border-r overflow-hidden flex flex-col">
-                <div className="p-4 border-b shrink-0">
-                  <label className="block text-sm font-medium text-gray-700">
-                    Target Column
-                  </label>
-                  <SearchableSelect
-                    options={dataset.columns.map((column) => ({
-                      value: column.name,
-                      label: column.name,
-                    }))}
-                    value={config.targetColumn || ""}
-                    onChange={(value) =>
-                      value && setTargetColumn(String(value))
-                    }
-                  />
+                <div className="p-4 border-b">
+                  <div className="flex border-b">
+                    <button
+                      onClick={() => setActiveColumnSubTab("target")}
+                      className={`flex items-center gap-2 px-4 py-2 border-b-2 ${
+                        activeColumnSubTab === "target"
+                          ? "border-blue-500 text-blue-600"
+                          : "border-transparent text-gray-500 hover:text-gray-700"
+                      }`}
+                    >
+                      <Target className="w-4 h-4" />
+                      Target Column
+                    </button>
+                    <button
+                      onClick={() => setActiveColumnSubTab("date")}
+                      className={`flex items-center gap-2 px-4 py-2 border-b-2 ${
+                        activeColumnSubTab === "date"
+                          ? "border-blue-500 text-blue-600"
+                          : "border-transparent text-gray-500 hover:text-gray-700"
+                      }`}
+                    >
+                      <Calendar className="w-4 h-4" />
+                      Date Column
+                    </button>
+                  </div>
+
+                  {activeColumnSubTab === "target" ? (
+                    <div className="mt-4">
+                      <SearchableSelect
+                        value={config.targetColumn || ""}
+                        onChange={(value) => setTargetColumn(value)}
+                        options={dataset.columns.map((column) => ({
+                          value: column.name,
+                          label: column.name,
+                        }))}
+                        placeholder="Select target column..."
+                      />
+                    </div>
+                  ) : (
+                    <div className="mt-4">
+                      {dateColumnOptions.length > 0 ? (
+                        <SearchableSelect
+                          options={dateColumnOptions}
+                          value={config.dateColumn}
+                          onChange={setDateColumn}
+                          placeholder="Select a date column..."
+                        />
+                      ) : (
+                        <div className="text-center py-4 text-gray-500 bg-gray-50 rounded-md">
+                          No date columns available
+                        </div>
+                      )}
+                    </div>
+                  )}
                 </div>
                 <div className="shrink-0">
                   <ColumnFilters

data/app/frontend/components/dataset/PreprocessingConfig.tsx

@@ -266,8 +266,13 @@ export function PreprocessingConfig({
     let content;
     if (strategy.method === 'most_frequent' && column.statistics?.raw.most_frequent_value) {
       content = `Most Frequent Value: ${column.statistics.raw.most_frequent_value}`
-    } else if (strategy.method === 'ffill' && column.statistics?.raw.last_value) {
-      content = `Last Value: ${column.statistics.raw.last_value}`
+    } else if (strategy.method === 'ffill') {
+      const lastValue = column.statistics?.raw.last_value;
+      if (lastValue !== undefined && lastValue !== null) {
+        content = `Forward Fill using Last Value: ${lastValue}`;
+      } else {
+        content = 'Set date column & apply preprocessing to see last value';
+      }
     } else if (strategy.method === 'median' && column.statistics?.raw?.median) {
       content = `Median: ${column.statistics.raw.median}`
     } else if (strategy.method === 'mean' && column.statistics?.raw?.mean) {

data/app/frontend/types/dataset.ts

@@ -94,6 +94,7 @@ export interface Dataset {
   needs_refresh: boolean;
   workflow_status: DatasetWorkflowStatus;
   target?: string;
+  date_column?: string;
   num_rows?: number;
   drop_cols?: string[];
   datasource_id: number;

data/app/helpers/easy_ml/application_helper.rb

@@ -2,9 +2,9 @@
 
 module EasyML
   module ApplicationHelper
-    # Override: Returns the engine assets manifest.
-    def easy_ml_manifest
-      ViteRuby.new(root: EasyML::Engine.root).manifest
+    def vite_manifest
+      # ViteRuby.new(root: EasyML::Engine.root).manifest
+      EasyML::Engine.vite_ruby.manifest
     end
 
     def prod_script_tags

data/app/models/easy_ml/column.rb

@@ -16,6 +16,7 @@
 #  statistics          :json
 #  created_at          :datetime         not null
 #  updated_at          :datetime         not null
+#  is_date_column      :boolean          default(FALSE)
 #
 module EasyML
   class Column < ActiveRecord::Base
@@ -29,12 +30,15 @@ module EasyML
     validates :name, uniqueness: { scope: :dataset_id }
 
     before_save :ensure_valid_datatype
+    after_create :set_date_column_if_date_splitter
+    after_save :handle_date_column_change
 
     # Scopes
     scope :visible, -> { where(hidden: false) }
     scope :numeric, -> { where(datatype: %w[float integer]) }
     scope :categorical, -> { where(datatype: %w[categorical string boolean]) }
     scope :datetime, -> { where(datatype: "datetime") }
+    scope :date_column, -> { where(is_date_column: true) }
 
     def datatype=(dtype)
       write_attribute(:datatype, dtype)
@@ -88,8 +92,26 @@ module EasyML
       dataset.preprocessor.statistics.dup.to_h.dig(name.to_sym, :allowed_categories).sort.concat(["other"])
     end
 
+    def date_column?
+      is_date_column
+    end
+
     private
 
+    def set_date_column_if_date_splitter
+      binding.pry
+    end
+
+    def handle_date_column_change
+      return unless saved_change_to_is_date_column? && is_date_column?
+
+      Column.transaction do
+        dataset.columns.where.not(id: id).update_all(is_date_column: false)
+        dataset.learn_statistics
+        dataset.columns.sync
+      end
+    end
+
     def ensure_valid_datatype
       return if datatype.blank?
 

data/app/models/easy_ml/column_history.rb

@@ -21,6 +21,7 @@
 #  history_ended_at    :datetime
 #  history_user_id     :integer
 #  snapshot_id         :string
+#  is_date_column      :boolean          default(FALSE)
 #
 module EasyML
   class ColumnHistory < ActiveRecord::Base

data/app/models/easy_ml/column_list.rb

@@ -9,6 +9,9 @@ module EasyML
         import_new(col_names, existing_columns)
         update_existing(existing_columns)
         delete_missing(existing_columns)
+        if existing_columns.none? # Totally new dataset
+          dataset.after_create_columns
+        end
       end
     end
 

data/app/models/easy_ml/dataset.rb

@@ -336,9 +336,12 @@ module EasyML
     end
 
     def learn_statistics
-      update(
-        statistics: EasyML::Data::StatisticsLearner.learn(raw, processed),
-      )
+      stats = {
+        raw: EasyML::Data::StatisticsLearner.learn(raw, self),
+      }
+      stats.merge!(processed: EasyML::Data::StatisticsLearner.learn(processed, self)) if processed.data.present?
+
+      update(statistics: stats)
     end
 
     def process_data
@@ -508,6 +511,10 @@ module EasyML
       @target ||= preloaded_columns.find(&:is_target)&.name
     end
 
+    def date_column
+      @date_column ||= columns.date_column.first
+    end
+
     def drop_cols
       @drop_cols ||= preloaded_columns.select(&:hidden).map(&:name)
     end
@@ -588,8 +595,18 @@ module EasyML
       self
     end
 
+    def after_create_columns
+      apply_date_splitter_config
+    end
+
     private
 
+    def apply_date_splitter_config
+      return unless splitter.date_splitter?
+
+      set_date_column(splitter.date_col)
+    end
+
     def preloaded_features
       @preloaded_features ||= features.includes(:dataset).load
     end
@@ -670,10 +687,8 @@ module EasyML
       end
     end
 
-    def fit(xs = nil)
-      xs = raw.train(all_columns: true) if xs.nil?
-
-      preprocessor.fit(xs)
+    def fit
+      preprocessor.fit(raw.train(all_columns: true))
       self.preprocessor_statistics = preprocessor.statistics
     end
 
@@ -712,6 +727,12 @@ module EasyML
       end
     end
 
+    def set_date_column(column_name)
+      return unless column_name.present?
+
+      columns.find_by(name: column_name).update(is_date_column: true)
+    end
+
     def apply_features(df, features = self.features)
       if features.nil? || features.empty?
         df
@@ -753,6 +774,7 @@ module EasyML
       EasyML::Data::Preprocessor.new(
         directory: Pathname.new(root_dir).append("preprocessor"),
         preprocessing_steps: preprocessing_steps,
+        dataset: self,
       ).tap do |preprocessor|
         preprocessor.statistics = preprocessor_statistics
       end

data/app/models/easy_ml/event_context.rb

@@ -5,6 +5,7 @@
 #  id         :bigint           not null, primary key
 #  event_id   :bigint           not null
 #  context    :jsonb            not null
+#  format     :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #

data/app/models/easy_ml/feature_history.rb

@@ -22,6 +22,7 @@
 #  history_ended_at   :datetime
 #  history_user_id    :integer
 #  snapshot_id        :string
+#  workflow_status    :string
 #
 module EasyML
   class FeatureHistory < ActiveRecord::Base

data/app/models/easy_ml/model_file.rb

@@ -4,7 +4,6 @@
 #
 #  id            :bigint           not null, primary key
 #  filename      :string           not null
-#  path          :string           not null
 #  configuration :json
 #  model_type    :string
 #  model_id      :bigint

data/app/models/easy_ml/model_file_history.rb

@@ -5,7 +5,6 @@
 #  id                 :bigint           not null, primary key
 #  model_file_id      :integer          not null
 #  filename           :string           not null
-#  path               :string           not null
 #  configuration      :json
 #  model_type         :string
 #  model_id           :integer

data/app/models/easy_ml/splitter.rb

@@ -52,6 +52,18 @@ module EasyML
       add_configuration_attributes attribute
     end
 
+    def date_splitter?
+      splitter_type == "date"
+    end
+
+    def random_splitter?
+      splitter_type == "random"
+    end
+
+    def predefined_splitter?
+      splitter_type == "predefined"
+    end
+
     def self.constants
       {
         SPLITTER_TYPES: SPLITTER_TYPES,

data/app/models/easy_ml/splitters/date_splitter.rb

@@ -86,6 +86,10 @@ module EasyML
       def today
         to_datetime(@today, default: UTC.today)
       end
+
+      def update_dataset_date_col
+        dataset.apply_date_splitter_config
+      end
     end
   end
 end

data/app/serializers/easy_ml/dataset_serializer.rb

@@ -29,6 +29,10 @@ module EasyML
     attributes :id, :name, :description, :target, :num_rows, :status,
                :datasource_id, :preprocessing_steps, :workflow_status, :statistics
 
+    attribute :date_column do |dataset|
+      dataset.date_column&.name
+    end
+
     attribute :splitter do |dataset|
       dataset.splitter
     end

data/config/initializers/dataframe.rb

@@ -0,0 +1,9 @@
+class Polars::DataFrame
+  def to_hash
+    if columns.count == 2
+      to_a.map(&:values).to_h
+    else
+      self
+    end
+  end
+end

data/lib/easy_ml/data/preprocessor.rb

@@ -18,16 +18,19 @@ module EasyML::Data
 
     PREPROCESSING_STRATEGIES = {
       float: [
+        { value: "ffill", label: "Forward Fill" },
         { value: "mean", label: "Mean" },
         { value: "median", label: "Median" },
         { value: "constant", label: "Constant Value" },
       ],
       integer: [
+        { value: "ffill", label: "Forward Fill" },
         { value: "mean", label: "Mean" },
         { value: "median", label: "Median" },
         { value: "constant", label: "Constant Value" },
       ],
       boolean: [
+        { value: "ffill", label: "Forward Fill" },
         { value: "most_frequent", label: "Most Frequent" },
         { value: "constant", label: "Constant Value" },
       ],
@@ -37,21 +40,24 @@ module EasyML::Data
         { value: "today", label: "Current Date" },
       ],
       string: [
+        { value: "ffill", label: "Forward Fill" },
         { value: "most_frequent", label: "Most Frequent" },
         { value: "constant", label: "Constant Value" },
       ],
       text: [
+        { value: "ffill", label: "Forward Fill" },
         { value: "most_frequent", label: "Most Frequent" },
         { value: "constant", label: "Constant Value" },
       ],
       categorical: [
+        { value: "ffill", label: "Forward Fill" },
         { value: "categorical", label: "Categorical" },
         { value: "most_frequent", label: "Most Frequent" },
         { value: "constant", label: "Constant Value" },
       ],
     }.freeze
 
-    attr_accessor :directory, :verbose, :imputers, :preprocessing_steps
+    attr_accessor :directory, :verbose, :imputers, :preprocessing_steps, :dataset
     attr_reader :statistics
 
     def initialize(options = {})
@@ -59,6 +65,7 @@ module EasyML::Data
       @verbose = options[:verbose]
       @imputers = options[:imputers]
       @preprocessing_steps = options[:preprocessing_steps]
+      @dataset = options[:dataset]
       @statistics = {}
     end
 
@@ -110,7 +117,7 @@ module EasyML::Data
       df = apply_clip(df, preprocessing_steps)
       allowed_categories = learn_categorical_min(df, preprocessing_steps)
 
-      self.statistics = StatisticsLearner.learn_df(df).deep_symbolize_keys
+      self.statistics = StatisticsLearner.learn_df(df, dataset: dataset).deep_symbolize_keys
 
       # Merge allowed categories into statistics
       allowed_categories.each do |col, categories|

data/lib/easy_ml/data/statistics_learner.rb

@@ -9,28 +9,55 @@ module EasyML::Data
       @verbose = options[:verbose]
     end
 
-    def self.learn(raw, processed)
-      output = { raw: learn_split(raw) }
-      output[:processed] = learn_split(processed) if processed.data.present?
-      output
+    def self.learn(df, dataset = nil)
+      new(df, dataset).learn
     end
 
-    def self.learn_split(split)
+    attr_reader :df, :dataset
+
+    def initialize(df, dataset)
+      @df = df
+      @dataset = dataset
+    end
+
+    def learn
+      learn_split(df)
+    end
+
+    def learn_split(split)
       df = split.read(:all)
       train_df = split.read(:train)
-      all_stats = learn_df(df)
-      train_stats = learn_df(train_df)
+      all_stats = learn_df(df, dataset: dataset)
+      train_stats = learn_df(train_df, dataset: dataset)
 
       all_stats.reduce({}) do |output, (k, _)|
         output.tap do
           output[k] = all_stats[k].slice(:num_rows, :null_count, :unique_count, :counts).merge!(
-            train_stats[k].slice(:mean, :median, :min, :max, :std, :last_value, :most_frequent_value)
+            train_stats[k].slice(:mean, :median, :min, :max, :std, :last_value, :most_frequent_value, :last_known_value)
           )
         end
       end
     end
 
-    def self.learn_df(df)
+    def last_known_value(df, col, date_col)
+      return nil if df.empty? || !df.columns.include?(date_col)
+
+      # Sort by date and get the last non-null value
+      sorted_df = df.sort(date_col, reverse: true)
+      last_value = sorted_df
+        .filter(Polars.col(col).is_not_null)
+        .select(col)
+        .head(1)
+        .item
+
+      last_value
+    end
+
+    def learn_df(df, dataset: nil)
+      self.class.learn_df(df, dataset: dataset)
+    end
+
+    def self.learn_df(df, dataset: nil)
       return if df.nil?
 
       base_stats = describe_to_h(df).deep_symbolize_keys
@@ -46,6 +73,10 @@ module EasyML::Data
           null_count: base_stats[col.to_sym][:null_count].to_i,
         }
 
+        if dataset&.date_column.present?
+          stats[col][:last_value] = last_value(df, col, dataset.date_column.name)
+        end
+
         # Add type-specific statistics
         case field_type
         when :integer, :float
@@ -77,6 +108,10 @@ module EasyML::Data
       col.match?(/^id$/) || col.match?(/.*_id/)
     end
 
+    def self.last_value(df, col, date_col)
+      df.filter(Polars.col(col).is_not_null).sort(date_col)[col][-1]
+    end
+
     def self.describe_to_h(df)
       init_h = df.describe.to_h
       rows = init_h.values.map(&:to_a)

data/lib/easy_ml/engine.rb

@@ -18,11 +18,16 @@ require "resque-batched-job"
 require "rake"
 require "resque/tasks"
 require "zhong"
+require "vite_ruby"
+require "vite_rails"
+require "dotenv"
 
 module EasyML
   class Engine < Rails::Engine
     isolate_namespace EasyML
 
+    Dotenv.load if File.exist?(".env")
+
     def root_dir
       Rails.root.join("easy_ml")
     end
@@ -118,7 +123,10 @@ module EasyML
       end
 
       puts "Running dev proxy"
-      config.app_middleware.insert_before 0, ViteRuby::DevServerProxy, ssl_verify_none: true, vite_ruby: vite_ruby
+      config.app_middleware.insert_before 0,
+        ViteRuby::DevServerProxy,
+        vite_ruby: vite_ruby,
+        ssl_verify_none: true
     else
       config.app_middleware.use(
         Rack::Static,