easy_ml 0.2.0.pre.rc39 → 0.2.0.pre.rc41

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +11 -9
  3. data/app/controllers/easy_ml/application_controller.rb +1 -1
  4. data/app/frontend/components/dataset/splitters/DateSplitter.tsx +4 -4
  5. data/app/frontend/components/dataset/splitters/types.ts +3 -3
  6. data/app/frontend/pages/NewDatasetPage.tsx +1 -1
  7. data/app/helpers/easy_ml/application_helper.rb +2 -2
  8. data/app/jobs/easy_ml/compute_feature_job.rb +54 -1
  9. data/app/models/concerns/easy_ml/dataframe_serialization.rb +30 -0
  10. data/app/models/easy_ml/dataset.rb +23 -22
  11. data/app/models/easy_ml/dataset_history.rb +1 -6
  12. data/app/models/easy_ml/datasources/polars_datasource.rb +4 -18
  13. data/app/models/easy_ml/event.rb +2 -1
  14. data/app/models/easy_ml/event_context.rb +58 -0
  15. data/app/models/easy_ml/feature.rb +40 -11
  16. data/app/models/easy_ml/model.rb +0 -1
  17. data/app/models/easy_ml/model_file.rb +7 -3
  18. data/app/models/easy_ml/splitter_history.rb +16 -0
  19. data/config/initializers/zhong.rb +4 -0
  20. data/lib/easy_ml/data/date_converter.rb +1 -0
  21. data/lib/easy_ml/data/polars_reader.rb +17 -4
  22. data/lib/easy_ml/data/statistics_learner.rb +1 -1
  23. data/lib/easy_ml/engine.rb +12 -1
  24. data/lib/easy_ml/pending_migrations.rb +19 -0
  25. data/lib/easy_ml/predict.rb +1 -3
  26. data/lib/easy_ml/railtie/generators/migration/migration_generator.rb +38 -157
  27. data/lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt +8 -0
  28. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt +4 -2
  29. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_columns.rb.tt +22 -20
  30. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_dataset_histories.rb.tt +5 -3
  31. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasets.rb.tt +26 -24
  32. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt +5 -3
  33. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt +12 -10
  34. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt +21 -19
  35. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_event_contexts.rb.tt +14 -0
  36. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt +16 -14
  37. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt +10 -8
  38. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_features.rb.tt +27 -25
  39. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_file_histories.rb.tt +5 -3
  40. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_files.rb.tt +13 -11
  41. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_histories.rb.tt +5 -3
  42. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_models.rb.tt +28 -26
  43. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_predictions.rb.tt +13 -11
  44. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_retraining_jobs.rb.tt +70 -67
  45. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_settings.rb.tt +6 -4
  46. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitter_histories.rb.tt +6 -4
  47. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitters.rb.tt +11 -9
  48. data/lib/easy_ml/railtie/templates/migration/create_easy_ml_tuner_jobs.rb.tt +34 -30
  49. data/lib/easy_ml/version.rb +1 -1
  50. data/lib/easy_ml.rb +1 -0
  51. data/public/easy_ml/assets/.vite/manifest.json +1 -1
  52. data/public/easy_ml/assets/assets/entrypoints/{Application.tsx-BRRjHz4-.js → Application.tsx-DF5SSkYi.js} +2 -2
  53. data/public/easy_ml/assets/assets/entrypoints/{Application.tsx-BRRjHz4-.js.map → Application.tsx-DF5SSkYi.js.map} +1 -1
  54. metadata +9 -18
@@ -1,32 +1,34 @@
1
1
  class CreateEasyMLFeatures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_features do |t|
4
- t.bigint :dataset_id, null: false
5
- t.string :name
6
- t.bigint :version
7
- t.string :feature_class, null: false
8
- t.integer :feature_position
9
- t.integer :batch_size
10
- t.boolean :needs_fit
11
- t.string :sha
12
- t.string :primary_key, array: true
13
- t.datetime :applied_at
14
- t.datetime :fit_at
15
- t.bigint :refresh_every
3
+ unless table_exists?(:easy_ml_features)
4
+ create_table :easy_ml_features do |t|
5
+ t.bigint :dataset_id, null: false
6
+ t.string :name
7
+ t.bigint :version
8
+ t.string :feature_class, null: false
9
+ t.integer :feature_position
10
+ t.integer :batch_size
11
+ t.boolean :needs_fit
12
+ t.string :sha
13
+ t.string :primary_key, array: true
14
+ t.datetime :applied_at
15
+ t.datetime :fit_at
16
+ t.bigint :refresh_every
16
17
 
17
- t.timestamps
18
+ t.timestamps
18
19
 
19
- t.index %i[dataset_id feature_position], name: "idx_features_on_dataset_and_position"
20
- t.index %i[dataset_id name], unique: true, name: "idx_features_on_dataset_and_name"
21
- t.index :feature_class
22
- t.index :applied_at
23
- t.index :name
24
- t.index :version
25
- t.index :sha
26
- t.index :batch_size
27
- t.index :needs_fit
28
- t.index :fit_at
29
- t.index :refresh_every
20
+ t.index %i[dataset_id feature_position], name: "idx_features_on_dataset_and_position"
21
+ t.index %i[dataset_id name], unique: true, name: "idx_features_on_dataset_and_name"
22
+ t.index :feature_class
23
+ t.index :applied_at
24
+ t.index :name
25
+ t.index :version
26
+ t.index :sha
27
+ t.index :batch_size
28
+ t.index :needs_fit
29
+ t.index :fit_at
30
+ t.index :refresh_every
31
+ end
30
32
  end
31
33
  end
32
34
  end
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
2
2
 
3
3
  class CreateEasyMLModelFileHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
4
4
  def change
5
- create_table :easy_ml_model_file_histories do |t|
6
- t.histories(foreign_key: :model_file_id)
5
+ unless table_exists?(:easy_ml_model_file_histories)
6
+ create_table :easy_ml_model_file_histories do |t|
7
+ t.histories(foreign_key: :model_file_id)
8
+ end
7
9
  end
8
10
  end
9
- end
11
+ end
@@ -1,17 +1,19 @@
1
1
  class CreateEasyMLModelFiles < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_model_files do |t|
4
- t.string :filename, null: false
5
- t.string :path, null: false
6
- t.json :configuration
7
- t.string :model_type
8
- t.bigint :model_id
9
- t.timestamps
3
+ unless table_exists?(:easy_ml_model_files)
4
+ create_table :easy_ml_model_files do |t|
5
+ t.string :filename, null: false
6
+ t.string :path, null: false
7
+ t.json :configuration
8
+ t.string :model_type
9
+ t.bigint :model_id
10
+ t.timestamps
10
11
 
11
- t.index :created_at
12
- t.index :filename
13
- t.index [:model_type]
14
- t.index :model_id
12
+ t.index :created_at
13
+ t.index :filename
14
+ t.index [:model_type]
15
+ t.index :model_id
16
+ end
15
17
  end
16
18
  end
17
19
  end
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
2
2
 
3
3
  class CreateEasyMLModelHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
4
4
  def change
5
- create_table :easy_ml_model_histories do |t|
6
- t.histories(foreign_key: :model_id)
5
+ unless table_exists?(:easy_ml_model_histories)
6
+ create_table :easy_ml_model_histories do |t|
7
+ t.histories(foreign_key: :model_id)
8
+ end
7
9
  end
8
10
  end
9
- end
11
+ end
@@ -1,34 +1,36 @@
1
1
  # lib/railtie/generators/templates/migration/create_easy_ml_models.rb.tt
2
2
  class CreateEasyMLModels < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
3
3
  def change
4
- create_table :easy_ml_models do |t|
5
- t.string :name, null: false
6
- t.string :model_type
7
- t.string :status
8
- t.bigint :dataset_id
9
- t.bigint :model_file_id
10
- t.json :configuration
11
- t.string :version, null: false
12
- t.string :root_dir
13
- t.json :file
14
- t.string :sha
15
- t.datetime :last_trained_at
16
- t.boolean :is_training
4
+ unless table_exists?(:easy_ml_models)
5
+ create_table :easy_ml_models do |t|
6
+ t.string :name, null: false
7
+ t.string :model_type
8
+ t.string :status
9
+ t.bigint :dataset_id
10
+ t.bigint :model_file_id
11
+ t.json :configuration
12
+ t.string :version, null: false
13
+ t.string :root_dir
14
+ t.json :file
15
+ t.string :sha
16
+ t.datetime :last_trained_at
17
+ t.boolean :is_training
17
18
 
18
- t.timestamps
19
+ t.timestamps
19
20
 
20
- t.index :created_at
21
- t.index :last_trained_at
22
- t.index :name
23
- t.index :version
24
- t.index :status
25
- t.index [:name, :status]
26
- t.index [:name, :version]
27
- t.index :dataset_id
28
- t.index :model_type
29
- t.index :model_file_id
30
- t.index :sha
31
- t.index :is_training
21
+ t.index :created_at
22
+ t.index :last_trained_at
23
+ t.index :name
24
+ t.index :version
25
+ t.index :status
26
+ t.index [:name, :status]
27
+ t.index [:name, :version]
28
+ t.index :dataset_id
29
+ t.index :model_type
30
+ t.index :model_file_id
31
+ t.index :sha
32
+ t.index :is_training
33
+ end
32
34
  end
33
35
  end
34
36
  end
@@ -1,17 +1,19 @@
1
1
  class CreateEasyMLPredictions < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_predictions do |t|
4
- t.bigint :model_id, null: false
5
- t.bigint :model_history_id
6
- t.string :prediction_type
7
- t.jsonb :prediction_value
8
- t.jsonb :raw_input
9
- t.jsonb :normalized_input
10
- t.timestamps
3
+ unless table_exists?(:easy_ml_predictions)
4
+ create_table :easy_ml_predictions do |t|
5
+ t.bigint :model_id, null: false
6
+ t.bigint :model_history_id
7
+ t.string :prediction_type
8
+ t.jsonb :prediction_value
9
+ t.jsonb :raw_input
10
+ t.jsonb :normalized_input
11
+ t.timestamps
11
12
 
12
- t.index :model_id
13
- t.index :model_history_id
14
- t.index :created_at
13
+ t.index :model_id
14
+ t.index :model_history_id
15
+ t.index :created_at
16
+ end
15
17
  end
16
18
  end
17
19
  end
@@ -1,77 +1,80 @@
1
1
  class CreateEasyMLRetrainingJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_retraining_jobs do |t|
4
- t.bigint :model_id
5
- t.string :frequency, null: false # day, week, month, hour
6
- t.json :at, null: false # hour of day (0-23)
7
- t.json :evaluator # Model evaluator
8
- t.boolean :tuning_enabled, default: false
9
- t.json :tuner_config # configuration for the tuner
10
- t.string :tuning_frequency # day, week, month, hour - when to run with tuner
11
- t.datetime :last_tuning_at # track last tuning run
12
- t.boolean :active, default: true
13
- t.string :status, default: "pending"
14
- t.datetime :last_run_at
15
- t.string :metric, null: false
16
- t.string :direction, null: false
17
- t.float :threshold, null: false
18
- t.boolean :auto_deploy, default: false
19
- t.boolean :batch_mode
20
- t.integer :batch_size
21
- t.integer :batch_overlap
22
- t.string :batch_key
3
+ unless table_exists?(:easy_ml_retraining_jobs)
4
+ create_table :easy_ml_retraining_jobs do |t|
5
+ t.bigint :model_id
6
+ t.string :frequency, null: false # day, week, month, hour
7
+ t.json :at, null: false # hour of day (0-23)
8
+ t.json :evaluator # Model evaluator
9
+ t.boolean :tuning_enabled, default: false
10
+ t.json :tuner_config # configuration for the tuner
11
+ t.string :tuning_frequency # day, week, month, hour - when to run with tuner
12
+ t.datetime :last_tuning_at # track last tuning run
13
+ t.boolean :active, default: true
14
+ t.string :status, default: "pending"
15
+ t.datetime :last_run_at
16
+ t.string :metric, null: false
17
+ t.string :direction, null: false
18
+ t.float :threshold, null: false
19
+ t.boolean :auto_deploy, default: false
20
+ t.boolean :batch_mode
21
+ t.integer :batch_size
22
+ t.integer :batch_overlap
23
+ t.string :batch_key
23
24
 
24
- t.timestamps
25
+ t.timestamps
25
26
 
26
- t.index :model_id
27
- t.index :active
28
- t.index :last_run_at
29
- t.index :last_tuning_at
30
- t.index :batch_mode
31
- t.index :auto_deploy
32
- t.index :tuning_enabled
33
- end
27
+ t.index :model_id
28
+ t.index :active
29
+ t.index :last_run_at
30
+ t.index :last_tuning_at
31
+ t.index :batch_mode
32
+ t.index :auto_deploy
33
+ t.index :tuning_enabled
34
+ end
34
35
 
35
- create_table :easy_ml_retraining_runs do |t|
36
- t.bigint :model_id
37
- t.bigint :model_history_id
38
- t.bigint :model_file_id
39
- t.bigint :retraining_job_id, null: false
40
- t.bigint :tuner_job_id, null: true
41
- t.string :status, default: 'pending'
42
- t.float :metric_value
43
- t.float :threshold
44
- t.string :trigger, default: 'manual'
45
- t.string :threshold_direction
46
- t.datetime :started_at
47
- t.datetime :completed_at
48
- t.text :error_message
49
- t.jsonb :metadata
50
- t.jsonb :metrics
51
- t.jsonb :best_params
52
- t.string :wandb_url
53
- t.string :snapshot_id
54
- t.boolean :deployable
55
- t.boolean :is_deploying
56
- t.boolean :deployed
57
- t.bigint :deploy_id
36
+ unless table_exists?(:easy_ml_retraining_runs)
37
+ create_table :easy_ml_retraining_runs do |t|
38
+ t.bigint :model_id
39
+ t.bigint :model_history_id
40
+ t.bigint :model_file_id
41
+ t.bigint :retraining_job_id, null: false
42
+ t.bigint :tuner_job_id, null: true
43
+ t.string :status, default: 'pending'
44
+ t.float :metric_value
45
+ t.float :threshold
46
+ t.string :trigger, default: 'manual'
47
+ t.string :threshold_direction
48
+ t.datetime :started_at
49
+ t.datetime :completed_at
50
+ t.text :error_message
51
+ t.jsonb :metadata
52
+ t.jsonb :metrics
53
+ t.jsonb :best_params
54
+ t.string :wandb_url
55
+ t.string :snapshot_id
56
+ t.boolean :deployable
57
+ t.boolean :is_deploying
58
+ t.boolean :deployed
59
+ t.bigint :deploy_id
58
60
 
59
- t.timestamps
61
+ t.timestamps
60
62
 
61
- t.index :status
62
- t.index :started_at
63
- t.index :completed_at
64
- t.index :created_at
65
- t.index :tuner_job_id
66
- t.index :retraining_job_id
67
- t.index :model_id
68
- t.index :trigger
69
- t.index :wandb_url
70
- t.index :snapshot_id
71
- t.index :deploy_id
72
- t.index :model_history_id
73
- t.index :deployable
74
- t.index :is_deploying
63
+ t.index :status
64
+ t.index :started_at
65
+ t.index :completed_at
66
+ t.index :created_at
67
+ t.index :tuner_job_id
68
+ t.index :retraining_job_id
69
+ t.index :model_id
70
+ t.index :trigger
71
+ t.index :wandb_url
72
+ t.index :snapshot_id
73
+ t.index :deploy_id
74
+ t.index :model_history_id
75
+ t.index :deployable
76
+ t.index :is_deploying
77
+ end
75
78
  end
76
79
  end
77
80
  end
@@ -1,9 +1,11 @@
1
1
  class CreateEasyMLSettings < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_settings do |t|
4
- t.json :configuration
3
+ unless table_exists?(:easy_ml_settings)
4
+ create_table :easy_ml_settings do |t|
5
+ t.json :configuration
5
6
 
6
- t.timestamps
7
+ t.timestamps
8
+ end
7
9
  end
8
10
  end
9
- end
11
+ end
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
2
2
 
3
3
  class CreateEasyMLSplitterHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
4
4
  def change
5
- create_table :easy_ml_splitter_histories do |t|
6
- t.histories(foreign_key: :splitter_id)
5
+ unless table_exists?(:easy_ml_splitter_histories)
6
+ create_table :easy_ml_splitter_histories do |t|
7
+ t.histories(foreign_key: :splitter_id)
8
+ end
7
9
  end
8
- end
9
- end
10
+ end
11
+ end
@@ -1,15 +1,17 @@
1
1
  class CreateEasyMLSplitters < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_splitters do |t|
4
- t.string :splitter_type, null: false
5
- t.json :configuration
6
- t.bigint :dataset_id, null: false
3
+ unless table_exists?(:easy_ml_splitters)
4
+ create_table :easy_ml_splitters do |t|
5
+ t.string :splitter_type, null: false
6
+ t.json :configuration
7
+ t.bigint :dataset_id, null: false
7
8
 
8
- t.timestamps
9
+ t.timestamps
9
10
 
10
- t.index :splitter_type
11
- t.index :created_at
12
- t.index :dataset_id
11
+ t.index :splitter_type
12
+ t.index :created_at
13
+ t.index :dataset_id
14
+ end
13
15
  end
14
16
  end
15
- end
17
+ end
@@ -1,40 +1,44 @@
1
1
  class CreateEasyMLTunerJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
2
2
  def change
3
- create_table :easy_ml_tuner_jobs do |t|
4
- t.json :config, null: false
5
- t.bigint :best_tuner_run_id
6
- t.bigint :model_id, null: false
7
- t.string :status
8
- t.string :direction, default: 'minimize'
9
- t.datetime :started_at
10
- t.datetime :completed_at
11
- t.jsonb :metadata
12
- t.string :wandb_url
3
+ unless table_exists?(:easy_ml_tuner_jobs)
4
+ create_table :easy_ml_tuner_jobs do |t|
5
+ t.json :config, null: false
6
+ t.bigint :best_tuner_run_id
7
+ t.bigint :model_id, null: false
8
+ t.string :status
9
+ t.string :direction, default: 'minimize'
10
+ t.datetime :started_at
11
+ t.datetime :completed_at
12
+ t.jsonb :metadata
13
+ t.string :wandb_url
13
14
 
14
- t.timestamps
15
+ t.timestamps
15
16
 
16
- t.index :status
17
- t.index :started_at
18
- t.index :completed_at
19
- t.index :model_id
20
- t.index :best_tuner_run_id
21
- t.index :wandb_url
17
+ t.index :status
18
+ t.index :started_at
19
+ t.index :completed_at
20
+ t.index :model_id
21
+ t.index :best_tuner_run_id
22
+ t.index :wandb_url
23
+ end
22
24
  end
23
25
 
24
- create_table :easy_ml_tuner_runs do |t|
25
- t.bigint :tuner_job_id, null: false
26
- t.json :hyperparameters, null: false
27
- t.float :value
28
- t.integer :trial_number
29
- t.string :status
30
- t.string :wandb_url
26
+ unless table_exists?(:easy_ml_tuner_runs)
27
+ create_table :easy_ml_tuner_runs do |t|
28
+ t.bigint :tuner_job_id, null: false
29
+ t.json :hyperparameters, null: false
30
+ t.float :value
31
+ t.integer :trial_number
32
+ t.string :status
33
+ t.string :wandb_url
31
34
 
32
- t.timestamps
35
+ t.timestamps
33
36
 
34
- t.index [:tuner_job_id, :value]
35
- t.index [:tuner_job_id, :trial_number], name: "idx_tuner_runs_and_trial_number"
36
- t.index :status
37
- t.index :wandb_url
37
+ t.index [:tuner_job_id, :value]
38
+ t.index [:tuner_job_id, :trial_number], name: "idx_tuner_runs_and_trial_number"
39
+ t.index :status
40
+ t.index :wandb_url
41
+ end
38
42
  end
39
43
  end
40
- end
44
+ end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module EasyML
4
- VERSION = "0.2.0-rc39"
4
+ VERSION = "0.2.0-rc41"
5
5
 
6
6
  module Version
7
7
  end
data/lib/easy_ml.rb CHANGED
@@ -24,4 +24,5 @@ module EasyML
24
24
  require_relative "easy_ml/feature_store"
25
25
  require_relative "easy_ml/core"
26
26
  require_relative "easy_ml/predict"
27
+ require_relative "easy_ml/pending_migrations"
27
28
  end
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "entrypoints/Application.tsx": {
3
- "file": "assets/entrypoints/Application.tsx-BRRjHz4-.js",
3
+ "file": "assets/entrypoints/Application.tsx-DF5SSkYi.js",
4
4
  "name": "entrypoints/Application.tsx",
5
5
  "src": "entrypoints/Application.tsx",
6
6
  "isEntry": true,