easy_ml 0.2.0.pre.rc40 → 0.2.0.pre.rc41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +11 -9
- data/app/jobs/easy_ml/compute_feature_job.rb +54 -1
- data/app/models/concerns/easy_ml/dataframe_serialization.rb +30 -0
- data/app/models/easy_ml/dataset.rb +23 -22
- data/app/models/easy_ml/dataset_history.rb +1 -6
- data/app/models/easy_ml/datasources/polars_datasource.rb +4 -18
- data/app/models/easy_ml/event.rb +2 -1
- data/app/models/easy_ml/event_context.rb +58 -0
- data/app/models/easy_ml/feature.rb +40 -11
- data/app/models/easy_ml/model.rb +0 -1
- data/app/models/easy_ml/model_file.rb +7 -3
- data/app/models/easy_ml/splitter_history.rb +16 -0
- data/config/initializers/zhong.rb +4 -0
- data/lib/easy_ml/data/date_converter.rb +1 -0
- data/lib/easy_ml/data/polars_reader.rb +17 -4
- data/lib/easy_ml/data/statistics_learner.rb +1 -1
- data/lib/easy_ml/engine.rb +11 -0
- data/lib/easy_ml/pending_migrations.rb +19 -0
- data/lib/easy_ml/predict.rb +1 -3
- data/lib/easy_ml/railtie/generators/migration/migration_generator.rb +38 -157
- data/lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt +8 -0
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt +4 -2
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_columns.rb.tt +22 -20
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_dataset_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasets.rb.tt +26 -24
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt +12 -10
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt +21 -19
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_event_contexts.rb.tt +14 -0
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt +16 -14
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt +10 -8
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_features.rb.tt +27 -25
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_file_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_files.rb.tt +13 -11
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_model_histories.rb.tt +5 -3
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_models.rb.tt +28 -26
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_predictions.rb.tt +13 -11
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_retraining_jobs.rb.tt +70 -67
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_settings.rb.tt +6 -4
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitter_histories.rb.tt +6 -4
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_splitters.rb.tt +11 -9
- data/lib/easy_ml/railtie/templates/migration/create_easy_ml_tuner_jobs.rb.tt +34 -30
- data/lib/easy_ml/version.rb +1 -1
- data/lib/easy_ml.rb +1 -0
- metadata +7 -16
@@ -1,34 +1,36 @@
|
|
1
1
|
# lib/railtie/generators/templates/migration/create_easy_ml_models.rb.tt
|
2
2
|
class CreateEasyMLModels < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
3
3
|
def change
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
4
|
+
unless table_exists?(:easy_ml_models)
|
5
|
+
create_table :easy_ml_models do |t|
|
6
|
+
t.string :name, null: false
|
7
|
+
t.string :model_type
|
8
|
+
t.string :status
|
9
|
+
t.bigint :dataset_id
|
10
|
+
t.bigint :model_file_id
|
11
|
+
t.json :configuration
|
12
|
+
t.string :version, null: false
|
13
|
+
t.string :root_dir
|
14
|
+
t.json :file
|
15
|
+
t.string :sha
|
16
|
+
t.datetime :last_trained_at
|
17
|
+
t.boolean :is_training
|
17
18
|
|
18
|
-
|
19
|
+
t.timestamps
|
19
20
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
21
|
+
t.index :created_at
|
22
|
+
t.index :last_trained_at
|
23
|
+
t.index :name
|
24
|
+
t.index :version
|
25
|
+
t.index :status
|
26
|
+
t.index [:name, :status]
|
27
|
+
t.index [:name, :version]
|
28
|
+
t.index :dataset_id
|
29
|
+
t.index :model_type
|
30
|
+
t.index :model_file_id
|
31
|
+
t.index :sha
|
32
|
+
t.index :is_training
|
33
|
+
end
|
32
34
|
end
|
33
35
|
end
|
34
36
|
end
|
@@ -1,17 +1,19 @@
|
|
1
1
|
class CreateEasyMLPredictions < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
3
|
+
unless table_exists?(:easy_ml_predictions)
|
4
|
+
create_table :easy_ml_predictions do |t|
|
5
|
+
t.bigint :model_id, null: false
|
6
|
+
t.bigint :model_history_id
|
7
|
+
t.string :prediction_type
|
8
|
+
t.jsonb :prediction_value
|
9
|
+
t.jsonb :raw_input
|
10
|
+
t.jsonb :normalized_input
|
11
|
+
t.timestamps
|
11
12
|
|
12
|
-
|
13
|
-
|
14
|
-
|
13
|
+
t.index :model_id
|
14
|
+
t.index :model_history_id
|
15
|
+
t.index :created_at
|
16
|
+
end
|
15
17
|
end
|
16
18
|
end
|
17
19
|
end
|
@@ -1,77 +1,80 @@
|
|
1
1
|
class CreateEasyMLRetrainingJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
3
|
+
unless table_exists?(:easy_ml_retraining_jobs)
|
4
|
+
create_table :easy_ml_retraining_jobs do |t|
|
5
|
+
t.bigint :model_id
|
6
|
+
t.string :frequency, null: false # day, week, month, hour
|
7
|
+
t.json :at, null: false # hour of day (0-23)
|
8
|
+
t.json :evaluator # Model evaluator
|
9
|
+
t.boolean :tuning_enabled, default: false
|
10
|
+
t.json :tuner_config # configuration for the tuner
|
11
|
+
t.string :tuning_frequency # day, week, month, hour - when to run with tuner
|
12
|
+
t.datetime :last_tuning_at # track last tuning run
|
13
|
+
t.boolean :active, default: true
|
14
|
+
t.string :status, default: "pending"
|
15
|
+
t.datetime :last_run_at
|
16
|
+
t.string :metric, null: false
|
17
|
+
t.string :direction, null: false
|
18
|
+
t.float :threshold, null: false
|
19
|
+
t.boolean :auto_deploy, default: false
|
20
|
+
t.boolean :batch_mode
|
21
|
+
t.integer :batch_size
|
22
|
+
t.integer :batch_overlap
|
23
|
+
t.string :batch_key
|
23
24
|
|
24
|
-
|
25
|
+
t.timestamps
|
25
26
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
27
|
+
t.index :model_id
|
28
|
+
t.index :active
|
29
|
+
t.index :last_run_at
|
30
|
+
t.index :last_tuning_at
|
31
|
+
t.index :batch_mode
|
32
|
+
t.index :auto_deploy
|
33
|
+
t.index :tuning_enabled
|
34
|
+
end
|
34
35
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
36
|
+
unless table_exists?(:easy_ml_retraining_runs)
|
37
|
+
create_table :easy_ml_retraining_runs do |t|
|
38
|
+
t.bigint :model_id
|
39
|
+
t.bigint :model_history_id
|
40
|
+
t.bigint :model_file_id
|
41
|
+
t.bigint :retraining_job_id, null: false
|
42
|
+
t.bigint :tuner_job_id, null: true
|
43
|
+
t.string :status, default: 'pending'
|
44
|
+
t.float :metric_value
|
45
|
+
t.float :threshold
|
46
|
+
t.string :trigger, default: 'manual'
|
47
|
+
t.string :threshold_direction
|
48
|
+
t.datetime :started_at
|
49
|
+
t.datetime :completed_at
|
50
|
+
t.text :error_message
|
51
|
+
t.jsonb :metadata
|
52
|
+
t.jsonb :metrics
|
53
|
+
t.jsonb :best_params
|
54
|
+
t.string :wandb_url
|
55
|
+
t.string :snapshot_id
|
56
|
+
t.boolean :deployable
|
57
|
+
t.boolean :is_deploying
|
58
|
+
t.boolean :deployed
|
59
|
+
t.bigint :deploy_id
|
58
60
|
|
59
|
-
|
61
|
+
t.timestamps
|
60
62
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
63
|
+
t.index :status
|
64
|
+
t.index :started_at
|
65
|
+
t.index :completed_at
|
66
|
+
t.index :created_at
|
67
|
+
t.index :tuner_job_id
|
68
|
+
t.index :retraining_job_id
|
69
|
+
t.index :model_id
|
70
|
+
t.index :trigger
|
71
|
+
t.index :wandb_url
|
72
|
+
t.index :snapshot_id
|
73
|
+
t.index :deploy_id
|
74
|
+
t.index :model_history_id
|
75
|
+
t.index :deployable
|
76
|
+
t.index :is_deploying
|
77
|
+
end
|
75
78
|
end
|
76
79
|
end
|
77
80
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
class CreateEasyMLSettings < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
3
|
+
unless table_exists?(:easy_ml_settings)
|
4
|
+
create_table :easy_ml_settings do |t|
|
5
|
+
t.json :configuration
|
5
6
|
|
6
|
-
|
7
|
+
t.timestamps
|
8
|
+
end
|
7
9
|
end
|
8
10
|
end
|
9
|
-
end
|
11
|
+
end
|
@@ -2,8 +2,10 @@ require "historiographer/postgres_migration"
|
|
2
2
|
|
3
3
|
class CreateEasyMLSplitterHistories < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
4
4
|
def change
|
5
|
-
|
6
|
-
|
5
|
+
unless table_exists?(:easy_ml_splitter_histories)
|
6
|
+
create_table :easy_ml_splitter_histories do |t|
|
7
|
+
t.histories(foreign_key: :splitter_id)
|
8
|
+
end
|
7
9
|
end
|
8
|
-
end
|
9
|
-
end
|
10
|
+
end
|
11
|
+
end
|
@@ -1,15 +1,17 @@
|
|
1
1
|
class CreateEasyMLSplitters < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
3
|
+
unless table_exists?(:easy_ml_splitters)
|
4
|
+
create_table :easy_ml_splitters do |t|
|
5
|
+
t.string :splitter_type, null: false
|
6
|
+
t.json :configuration
|
7
|
+
t.bigint :dataset_id, null: false
|
7
8
|
|
8
|
-
|
9
|
+
t.timestamps
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
11
|
+
t.index :splitter_type
|
12
|
+
t.index :created_at
|
13
|
+
t.index :dataset_id
|
14
|
+
end
|
13
15
|
end
|
14
16
|
end
|
15
|
-
end
|
17
|
+
end
|
@@ -1,40 +1,44 @@
|
|
1
1
|
class CreateEasyMLTunerJobs < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
2
2
|
def change
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
3
|
+
unless table_exists?(:easy_ml_tuner_jobs)
|
4
|
+
create_table :easy_ml_tuner_jobs do |t|
|
5
|
+
t.json :config, null: false
|
6
|
+
t.bigint :best_tuner_run_id
|
7
|
+
t.bigint :model_id, null: false
|
8
|
+
t.string :status
|
9
|
+
t.string :direction, default: 'minimize'
|
10
|
+
t.datetime :started_at
|
11
|
+
t.datetime :completed_at
|
12
|
+
t.jsonb :metadata
|
13
|
+
t.string :wandb_url
|
13
14
|
|
14
|
-
|
15
|
+
t.timestamps
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
17
|
+
t.index :status
|
18
|
+
t.index :started_at
|
19
|
+
t.index :completed_at
|
20
|
+
t.index :model_id
|
21
|
+
t.index :best_tuner_run_id
|
22
|
+
t.index :wandb_url
|
23
|
+
end
|
22
24
|
end
|
23
25
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
unless table_exists?(:easy_ml_tuner_runs)
|
27
|
+
create_table :easy_ml_tuner_runs do |t|
|
28
|
+
t.bigint :tuner_job_id, null: false
|
29
|
+
t.json :hyperparameters, null: false
|
30
|
+
t.float :value
|
31
|
+
t.integer :trial_number
|
32
|
+
t.string :status
|
33
|
+
t.string :wandb_url
|
31
34
|
|
32
|
-
|
35
|
+
t.timestamps
|
33
36
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
37
|
+
t.index [:tuner_job_id, :value]
|
38
|
+
t.index [:tuner_job_id, :trial_number], name: "idx_tuner_runs_and_trial_number"
|
39
|
+
t.index :status
|
40
|
+
t.index :wandb_url
|
41
|
+
end
|
38
42
|
end
|
39
43
|
end
|
40
|
-
end
|
44
|
+
end
|
data/lib/easy_ml/version.rb
CHANGED
data/lib/easy_ml.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: easy_ml
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.0.pre.
|
4
|
+
version: 0.2.0.pre.rc41
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Brett Shollenberger
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2025-01-
|
11
|
+
date: 2025-01-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -402,20 +402,6 @@ dependencies:
|
|
402
402
|
- - ">="
|
403
403
|
- !ruby/object:Gem::Version
|
404
404
|
version: '0'
|
405
|
-
- !ruby/object:Gem::Dependency
|
406
|
-
name: sprockets-rails
|
407
|
-
requirement: !ruby/object:Gem::Requirement
|
408
|
-
requirements:
|
409
|
-
- - ">="
|
410
|
-
- !ruby/object:Gem::Version
|
411
|
-
version: '0'
|
412
|
-
type: :development
|
413
|
-
prerelease: false
|
414
|
-
version_requirements: !ruby/object:Gem::Requirement
|
415
|
-
requirements:
|
416
|
-
- - ">="
|
417
|
-
- !ruby/object:Gem::Version
|
418
|
-
version: '0'
|
419
405
|
- !ruby/object:Gem::Dependency
|
420
406
|
name: tailwindcss-rails
|
421
407
|
requirement: !ruby/object:Gem::Requirement
|
@@ -543,6 +529,7 @@ files:
|
|
543
529
|
- app/jobs/easy_ml/schedule_retraining_job.rb
|
544
530
|
- app/jobs/easy_ml/sync_datasource_job.rb
|
545
531
|
- app/jobs/easy_ml/training_job.rb
|
532
|
+
- app/models/concerns/easy_ml/dataframe_serialization.rb
|
546
533
|
- app/models/easy_ml/adapters/base_adapter.rb
|
547
534
|
- app/models/easy_ml/adapters/polars_adapter.rb
|
548
535
|
- app/models/easy_ml/cleaner.rb
|
@@ -561,6 +548,7 @@ files:
|
|
561
548
|
- app/models/easy_ml/datasources/s3_datasource.rb
|
562
549
|
- app/models/easy_ml/deploy.rb
|
563
550
|
- app/models/easy_ml/event.rb
|
551
|
+
- app/models/easy_ml/event_context.rb
|
564
552
|
- app/models/easy_ml/feature.rb
|
565
553
|
- app/models/easy_ml/feature_history.rb
|
566
554
|
- app/models/easy_ml/model.rb
|
@@ -650,8 +638,10 @@ files:
|
|
650
638
|
- lib/easy_ml/feature_store.rb
|
651
639
|
- lib/easy_ml/features.rb
|
652
640
|
- lib/easy_ml/logging.rb
|
641
|
+
- lib/easy_ml/pending_migrations.rb
|
653
642
|
- lib/easy_ml/predict.rb
|
654
643
|
- lib/easy_ml/railtie/generators/migration/migration_generator.rb
|
644
|
+
- lib/easy_ml/railtie/templates/migration/add_workflow_status_to_easy_ml_features.rb.tt
|
655
645
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_column_histories.rb.tt
|
656
646
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_columns.rb.tt
|
657
647
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_dataset_histories.rb.tt
|
@@ -659,6 +649,7 @@ files:
|
|
659
649
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_datasource_histories.rb.tt
|
660
650
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_datasources.rb.tt
|
661
651
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_deploys.rb.tt
|
652
|
+
- lib/easy_ml/railtie/templates/migration/create_easy_ml_event_contexts.rb.tt
|
662
653
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_events.rb.tt
|
663
654
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_feature_histories.rb.tt
|
664
655
|
- lib/easy_ml/railtie/templates/migration/create_easy_ml_features.rb.tt
|