easy_ml 0.2.0.pre.rc39 → 0.2.0.pre.rc41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +11 -9
- data/app/controllers/easy_ml/application_controller.rb +1 -1
- data/app/frontend/components/dataset/splitters/DateSplitter.tsx +4 -4
- data/app/frontend/components/dataset/splitters/types.ts +3 -3
- data/app/frontend/pages/NewDatasetPage.tsx +1 -1
- data/app/helpers/easy_ml/application_helper.rb +2 -2
- data/app/jobs/easy_ml/compute_feature_job.rb +54 -1
- data/app/models/concerns/easy_ml/dataframe_serialization.rb +30 -0
- data/app/models/easy_ml/dataset.rb +23 -22
- data/app/models/easy_ml/dataset_history.rb +1 -6
- data/app/models/easy_ml/datasources/polars_datasource.rb +4 -18
- data/app/models/easy_ml/event.rb +2 -1
- data/app/models/easy_ml/event_context.rb +58 -0
- data/app/models/easy_ml/feature.rb +40 -11
- data/app/models/easy_ml/model.rb +0 -1
- data/app/models/easy_ml/model_file.rb +7 -3
- data/app/models/easy_ml/splitter_history.rb +16 -0
- data/config/initializers/zhong.rb +4 -0
- data/lib/easy_ml/data/date_converter.rb +1 -0
- data/lib/easy_ml/data/polars_reader.rb +17 -4
- data/lib/easy_ml/data/statistics_learner.rb +1 -1
- data/lib/easy_ml/engine.rb +12 -1
- data/lib/easy_ml/pending_migrations.rb +19 -0
- data/lib/easy_ml/predict.rb +1 -3
- data/lib/easy_ml/railtie/generators/migration/migration_generator.rb +38 -157
- data/lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt +8 -0
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt +4 -2
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_columns.rb.tt +22 -20
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_dataset_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasets.rb.tt +26 -24
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt +12 -10
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt +21 -19
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_event_contexts.rb.tt +14 -0
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt +16 -14
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt +10 -8
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_features.rb.tt +27 -25
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_file_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_files.rb.tt +13 -11
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_models.rb.tt +28 -26
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_predictions.rb.tt +13 -11
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_retraining_jobs.rb.tt +70 -67
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_settings.rb.tt +6 -4
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitter_histories.rb.tt +6 -4
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitters.rb.tt +11 -9
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_tuner_jobs.rb.tt +34 -30
- data/lib/easy_ml/version.rb +1 -1
- data/lib/easy_ml.rb +1 -0
- data/public/easy_ml/assets/.vite/manifest.json +1 -1
- data/public/easy_ml/assets/assets/entrypoints/{Application.tsx-BRRjHz4-.js → Application.tsx-DF5SSkYi.js} +2 -2
- data/public/easy_ml/assets/assets/entrypoints/{Application.tsx-BRRjHz4-.js.map → Application.tsx-DF5SSkYi.js.map} +1 -1
- metadata +9 -18
@@ -1,32 +1,34 @@
|
|
1
1
|
class CreateEasyMLFeatures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
3
|
+
unless table_exists?(:easy_ml_features)
|
4
|
+
create_table :easy_ml_features do |t|
|
5
|
+
t.bigint :dataset_id, null: false
|
6
|
+
t.string :name
|
7
|
+
t.bigint :version
|
8
|
+
t.string :feature_class, null: false
|
9
|
+
t.integer :feature_position
|
10
|
+
t.integer :batch_size
|
11
|
+
t.boolean :needs_fit
|
12
|
+
t.string :sha
|
13
|
+
t.string :primary_key, array: true
|
14
|
+
t.datetime :applied_at
|
15
|
+
t.datetime :fit_at
|
16
|
+
t.bigint :refresh_every
|
16
17
|
|
17
|
-
|
18
|
+
t.timestamps
|
18
19
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
20
|
+
t.index %i[dataset_id feature_position], name: "idx_features_on_dataset_and_position"
|
21
|
+
t.index %i[dataset_id name], unique: true, name: "idx_features_on_dataset_and_name"
|
22
|
+
t.index :feature_class
|
23
|
+
t.index :applied_at
|
24
|
+
t.index :name
|
25
|
+
t.index :version
|
26
|
+
t.index :sha
|
27
|
+
t.index :batch_size
|
28
|
+
t.index :needs_fit
|
29
|
+
t.index :fit_at
|
30
|
+
t.index :refresh_every
|
31
|
+
end
|
30
32
|
end
|
31
33
|
end
|
32
34
|
end
|
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
|
|
2
2
|
|
3
3
|
class CreateEasyMLModelFileHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
4
4
|
def change
|
5
|
-
|
6
|
-
|
5
|
+
unless table_exists?(:easy_ml_model_file_histories)
|
6
|
+
create_table :easy_ml_model_file_histories do |t|
|
7
|
+
t.histories(foreign_key: :model_file_id)
|
8
|
+
end
|
7
9
|
end
|
8
10
|
end
|
9
|
-
end
|
11
|
+
end
|
@@ -1,17 +1,19 @@
|
|
1
1
|
class CreateEasyMLModelFiles < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
3
|
+
unless table_exists?(:easy_ml_model_files)
|
4
|
+
create_table :easy_ml_model_files do |t|
|
5
|
+
t.string :filename, null: false
|
6
|
+
t.string :path, null: false
|
7
|
+
t.json :configuration
|
8
|
+
t.string :model_type
|
9
|
+
t.bigint :model_id
|
10
|
+
t.timestamps
|
10
11
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
t.index :created_at
|
13
|
+
t.index :filename
|
14
|
+
t.index [:model_type]
|
15
|
+
t.index :model_id
|
16
|
+
end
|
15
17
|
end
|
16
18
|
end
|
17
19
|
end
|
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
|
|
2
2
|
|
3
3
|
class CreateEasyMLModelHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
4
4
|
def change
|
5
|
-
|
6
|
-
|
5
|
+
unless table_exists?(:easy_ml_model_histories)
|
6
|
+
create_table :easy_ml_model_histories do |t|
|
7
|
+
t.histories(foreign_key: :model_id)
|
8
|
+
end
|
7
9
|
end
|
8
10
|
end
|
9
|
-
end
|
11
|
+
end
|
@@ -1,34 +1,36 @@
|
|
1
1
|
# lib/railtie/generators/templates/migration/create_easy_ml_models.rb.tt
|
2
2
|
class CreateEasyMLModels < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
3
3
|
def change
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
4
|
+
unless table_exists?(:easy_ml_models)
|
5
|
+
create_table :easy_ml_models do |t|
|
6
|
+
t.string :name, null: false
|
7
|
+
t.string :model_type
|
8
|
+
t.string :status
|
9
|
+
t.bigint :dataset_id
|
10
|
+
t.bigint :model_file_id
|
11
|
+
t.json :configuration
|
12
|
+
t.string :version, null: false
|
13
|
+
t.string :root_dir
|
14
|
+
t.json :file
|
15
|
+
t.string :sha
|
16
|
+
t.datetime :last_trained_at
|
17
|
+
t.boolean :is_training
|
17
18
|
|
18
|
-
|
19
|
+
t.timestamps
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
21
|
+
t.index :created_at
|
22
|
+
t.index :last_trained_at
|
23
|
+
t.index :name
|
24
|
+
t.index :version
|
25
|
+
t.index :status
|
26
|
+
t.index [:name, :status]
|
27
|
+
t.index [:name, :version]
|
28
|
+
t.index :dataset_id
|
29
|
+
t.index :model_type
|
30
|
+
t.index :model_file_id
|
31
|
+
t.index :sha
|
32
|
+
t.index :is_training
|
33
|
+
end
|
32
34
|
end
|
33
35
|
end
|
34
36
|
end
|
@@ -1,17 +1,19 @@
|
|
1
1
|
class CreateEasyMLPredictions < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
3
|
+
unless table_exists?(:easy_ml_predictions)
|
4
|
+
create_table :easy_ml_predictions do |t|
|
5
|
+
t.bigint :model_id, null: false
|
6
|
+
t.bigint :model_history_id
|
7
|
+
t.string :prediction_type
|
8
|
+
t.jsonb :prediction_value
|
9
|
+
t.jsonb :raw_input
|
10
|
+
t.jsonb :normalized_input
|
11
|
+
t.timestamps
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
t.index :model_id
|
14
|
+
t.index :model_history_id
|
15
|
+
t.index :created_at
|
16
|
+
end
|
15
17
|
end
|
16
18
|
end
|
17
19
|
end
|
@@ -1,77 +1,80 @@
|
|
1
1
|
class CreateEasyMLRetrainingJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
3
|
+
unless table_exists?(:easy_ml_retraining_jobs)
|
4
|
+
create_table :easy_ml_retraining_jobs do |t|
|
5
|
+
t.bigint :model_id
|
6
|
+
t.string :frequency, null: false # day, week, month, hour
|
7
|
+
t.json :at, null: false # hour of day (0-23)
|
8
|
+
t.json :evaluator # Model evaluator
|
9
|
+
t.boolean :tuning_enabled, default: false
|
10
|
+
t.json :tuner_config # configuration for the tuner
|
11
|
+
t.string :tuning_frequency # day, week, month, hour - when to run with tuner
|
12
|
+
t.datetime :last_tuning_at # track last tuning run
|
13
|
+
t.boolean :active, default: true
|
14
|
+
t.string :status, default: "pending"
|
15
|
+
t.datetime :last_run_at
|
16
|
+
t.string :metric, null: false
|
17
|
+
t.string :direction, null: false
|
18
|
+
t.float :threshold, null: false
|
19
|
+
t.boolean :auto_deploy, default: false
|
20
|
+
t.boolean :batch_mode
|
21
|
+
t.integer :batch_size
|
22
|
+
t.integer :batch_overlap
|
23
|
+
t.string :batch_key
|
23
24
|
|
24
|
-
|
25
|
+
t.timestamps
|
25
26
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
27
|
+
t.index :model_id
|
28
|
+
t.index :active
|
29
|
+
t.index :last_run_at
|
30
|
+
t.index :last_tuning_at
|
31
|
+
t.index :batch_mode
|
32
|
+
t.index :auto_deploy
|
33
|
+
t.index :tuning_enabled
|
34
|
+
end
|
34
35
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
36
|
+
unless table_exists?(:easy_ml_retraining_runs)
|
37
|
+
create_table :easy_ml_retraining_runs do |t|
|
38
|
+
t.bigint :model_id
|
39
|
+
t.bigint :model_history_id
|
40
|
+
t.bigint :model_file_id
|
41
|
+
t.bigint :retraining_job_id, null: false
|
42
|
+
t.bigint :tuner_job_id, null: true
|
43
|
+
t.string :status, default: 'pending'
|
44
|
+
t.float :metric_value
|
45
|
+
t.float :threshold
|
46
|
+
t.string :trigger, default: 'manual'
|
47
|
+
t.string :threshold_direction
|
48
|
+
t.datetime :started_at
|
49
|
+
t.datetime :completed_at
|
50
|
+
t.text :error_message
|
51
|
+
t.jsonb :metadata
|
52
|
+
t.jsonb :metrics
|
53
|
+
t.jsonb :best_params
|
54
|
+
t.string :wandb_url
|
55
|
+
t.string :snapshot_id
|
56
|
+
t.boolean :deployable
|
57
|
+
t.boolean :is_deploying
|
58
|
+
t.boolean :deployed
|
59
|
+
t.bigint :deploy_id
|
58
60
|
|
59
|
-
|
61
|
+
t.timestamps
|
60
62
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
63
|
+
t.index :status
|
64
|
+
t.index :started_at
|
65
|
+
t.index :completed_at
|
66
|
+
t.index :created_at
|
67
|
+
t.index :tuner_job_id
|
68
|
+
t.index :retraining_job_id
|
69
|
+
t.index :model_id
|
70
|
+
t.index :trigger
|
71
|
+
t.index :wandb_url
|
72
|
+
t.index :snapshot_id
|
73
|
+
t.index :deploy_id
|
74
|
+
t.index :model_history_id
|
75
|
+
t.index :deployable
|
76
|
+
t.index :is_deploying
|
77
|
+
end
|
75
78
|
end
|
76
79
|
end
|
77
80
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
class CreateEasyMLSettings < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
3
|
+
unless table_exists?(:easy_ml_settings)
|
4
|
+
create_table :easy_ml_settings do |t|
|
5
|
+
t.json :configuration
|
5
6
|
|
6
|
-
|
7
|
+
t.timestamps
|
8
|
+
end
|
7
9
|
end
|
8
10
|
end
|
9
|
-
end
|
11
|
+
end
|
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
|
|
2
2
|
|
3
3
|
class CreateEasyMLSplitterHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
4
4
|
def change
|
5
|
-
|
6
|
-
|
5
|
+
unless table_exists?(:easy_ml_splitter_histories)
|
6
|
+
create_table :easy_ml_splitter_histories do |t|
|
7
|
+
t.histories(foreign_key: :splitter_id)
|
8
|
+
end
|
7
9
|
end
|
8
|
-
end
|
9
|
-
end
|
10
|
+
end
|
11
|
+
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
class CreateEasyMLSplitters < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
unless table_exists?(:easy_ml_splitters)
|
4
|
+
create_table :easy_ml_splitters do |t|
|
5
|
+
t.string :splitter_type, null: false
|
6
|
+
t.json :configuration
|
7
|
+
t.bigint :dataset_id, null: false
|
7
8
|
|
8
|
-
|
9
|
+
t.timestamps
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
t.index :splitter_type
|
12
|
+
t.index :created_at
|
13
|
+
t.index :dataset_id
|
14
|
+
end
|
13
15
|
end
|
14
16
|
end
|
15
|
-
end
|
17
|
+
end
|
@@ -1,40 +1,44 @@
|
|
1
1
|
class CreateEasyMLTunerJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
unless table_exists?(:easy_ml_tuner_jobs)
|
4
|
+
create_table :easy_ml_tuner_jobs do |t|
|
5
|
+
t.json :config, null: false
|
6
|
+
t.bigint :best_tuner_run_id
|
7
|
+
t.bigint :model_id, null: false
|
8
|
+
t.string :status
|
9
|
+
t.string :direction, default: 'minimize'
|
10
|
+
t.datetime :started_at
|
11
|
+
t.datetime :completed_at
|
12
|
+
t.jsonb :metadata
|
13
|
+
t.string :wandb_url
|
13
14
|
|
14
|
-
|
15
|
+
t.timestamps
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
17
|
+
t.index :status
|
18
|
+
t.index :started_at
|
19
|
+
t.index :completed_at
|
20
|
+
t.index :model_id
|
21
|
+
t.index :best_tuner_run_id
|
22
|
+
t.index :wandb_url
|
23
|
+
end
|
22
24
|
end
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
unless table_exists?(:easy_ml_tuner_runs)
|
27
|
+
create_table :easy_ml_tuner_runs do |t|
|
28
|
+
t.bigint :tuner_job_id, null: false
|
29
|
+
t.json :hyperparameters, null: false
|
30
|
+
t.float :value
|
31
|
+
t.integer :trial_number
|
32
|
+
t.string :status
|
33
|
+
t.string :wandb_url
|
31
34
|
|
32
|
-
|
35
|
+
t.timestamps
|
33
36
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
37
|
+
t.index [:tuner_job_id, :value]
|
38
|
+
t.index [:tuner_job_id, :trial_number], name: "idx_tuner_runs_and_trial_number"
|
39
|
+
t.index :status
|
40
|
+
t.index :wandb_url
|
41
|
+
end
|
38
42
|
end
|
39
43
|
end
|
40
|
-
end
|
44
|
+
end
|
data/lib/easy_ml/version.rb
CHANGED
data/lib/easy_ml.rb
CHANGED