ragdoll 0.1.0 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +318 -40
  3. data/Rakefile +66 -4
  4. data/app/jobs/ragdoll/extract_keywords_job.rb +28 -0
  5. data/app/jobs/ragdoll/extract_text_job.rb +38 -0
  6. data/app/jobs/ragdoll/generate_embeddings_job.rb +28 -0
  7. data/app/jobs/ragdoll/generate_summary_job.rb +25 -0
  8. data/app/lib/ragdoll/metadata_schemas.rb +332 -0
  9. data/app/models/ragdoll/audio_content.rb +142 -0
  10. data/app/models/ragdoll/content.rb +95 -0
  11. data/app/models/ragdoll/document.rb +606 -4
  12. data/app/models/ragdoll/embedding.rb +172 -5
  13. data/app/models/ragdoll/image_content.rb +194 -0
  14. data/app/models/ragdoll/text_content.rb +137 -0
  15. data/app/services/ragdoll/configuration_service.rb +113 -0
  16. data/app/services/ragdoll/document_management.rb +108 -0
  17. data/app/services/ragdoll/document_processor.rb +342 -0
  18. data/app/services/ragdoll/embedding_service.rb +202 -0
  19. data/app/services/ragdoll/image_description_service.rb +230 -0
  20. data/app/services/ragdoll/metadata_generator.rb +329 -0
  21. data/app/services/ragdoll/model_resolver.rb +72 -0
  22. data/app/services/ragdoll/search_engine.rb +51 -0
  23. data/app/services/ragdoll/text_chunker.rb +208 -0
  24. data/app/services/ragdoll/text_generation_service.rb +355 -0
  25. data/db/migrate/001_enable_postgresql_extensions.rb +23 -0
  26. data/db/migrate/004_create_ragdoll_documents.rb +70 -0
  27. data/db/migrate/005_create_ragdoll_embeddings.rb +41 -0
  28. data/db/migrate/006_create_ragdoll_contents.rb +47 -0
  29. data/lib/ragdoll/core/client.rb +306 -0
  30. data/lib/ragdoll/core/configuration.rb +257 -0
  31. data/lib/ragdoll/core/database.rb +141 -0
  32. data/lib/ragdoll/core/errors.rb +11 -0
  33. data/lib/ragdoll/core/model.rb +45 -0
  34. data/lib/ragdoll/core/shrine_config.rb +71 -0
  35. data/lib/ragdoll/core/version.rb +8 -0
  36. data/lib/ragdoll/core.rb +91 -0
  37. data/lib/ragdoll-core.rb +3 -0
  38. data/lib/ragdoll.rb +243 -6
  39. data/lib/tasks/annotate.rake +126 -0
  40. data/lib/tasks/db.rake +338 -0
  41. metadata +42 -35
  42. data/config/initializers/ragdoll.rb +0 -6
  43. data/config/routes.rb +0 -5
  44. data/db/migrate/20250218123456_create_documents.rb +0 -20
  45. data/lib/config/database.yml +0 -28
  46. data/lib/config/ragdoll.yml +0 -31
  47. data/lib/ragdoll/engine.rb +0 -16
  48. data/lib/ragdoll/import_job.rb +0 -15
  49. data/lib/ragdoll/ingestion.rb +0 -30
  50. data/lib/ragdoll/search.rb +0 -18
  51. data/lib/ragdoll/version.rb +0 -7
  52. data/lib/tasks/import_task.thor +0 -32
  53. data/lib/tasks/jobs_task.thor +0 -40
  54. data/lib/tasks/ragdoll_tasks.thor +0 -7
  55. data/lib/tasks/search_task.thor +0 -55
@@ -0,0 +1,126 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "annotate"
4
+ require "annotate/annotate_models"
5
+
6
+ # Define environment task for gem context
7
+ task :environment do
8
+ # Load the gem's environment
9
+ require_relative "../ragdoll-core"
10
+
11
+ # Set up database connection
12
+ begin
13
+ # Load all models to ensure they're available
14
+ Dir[File.join(__dir__, "../ragdoll/core/models/*.rb")].each { |file| require file }
15
+
16
+ # Set up database connection with default config
17
+ Ragdoll::Core::Database.setup({
18
+ adapter: "postgresql",
19
+ database: "ragdoll_development",
20
+ username: "ragdoll",
21
+ password: ENV.fetch("RAGDOLL_DATABASE_PASSWORD", ENV["DATABASE_PASSWORD"]),
22
+ host: "localhost",
23
+ port: 5432,
24
+ auto_migrate: false # Don't auto-migrate during annotation
25
+ })
26
+
27
+ puts "✅ Connected to database: ragdoll_development"
28
+ rescue StandardError => e
29
+ puts "❌ Database connection failed: #{e.message}"
30
+ puts " Annotations will be based on model definitions only"
31
+ end
32
+ end
33
+
34
+ namespace :annotate do
35
+ task :models do
36
+ AnnotateModels.do_annotations({
37
+ "models" => "true",
38
+ "position_in_class" => "before",
39
+ "show_foreign_keys" => "true",
40
+ "show_indexes" => "true",
41
+ "model_dir" => "lib/ragdoll/core/models",
42
+ "exclude_tests" => "true",
43
+ "exclude_fixtures" => "true",
44
+ "exclude_factories" => "true",
45
+ "exclude_serializers" => "true",
46
+ "exclude_scaffolds" => "true",
47
+ "exclude_controllers" => "true",
48
+ "exclude_helpers" => "true"
49
+ })
50
+ end
51
+ end
52
+
53
+ task :set_annotation_options do
54
+ # You can override any of these by setting an environment variable of the
55
+ # same name.
56
+ Annotate.set_defaults(
57
+ "active_admin" => "false",
58
+ "additional_file_patterns" => [],
59
+ "routes" => "false",
60
+ "models" => "true",
61
+ "position_in_routes" => "before",
62
+ "position_in_class" => "before",
63
+ "position_in_test" => "before",
64
+ "position_in_fixture" => "before",
65
+ "position_in_factory" => "before",
66
+ "position_in_serializer" => "before",
67
+ "show_foreign_keys" => "true",
68
+ "show_complete_foreign_keys" => "false",
69
+ "show_indexes" => "true",
70
+ "simple_indexes" => "false",
71
+ "model_dir" => "lib/ragdoll/core/models",
72
+ "root_dir" => "",
73
+ "include_version" => "false",
74
+ "require" => "",
75
+ "exclude_tests" => "false",
76
+ "exclude_fixtures" => "false",
77
+ "exclude_factories" => "false",
78
+ "exclude_serializers" => "false",
79
+ "exclude_scaffolds" => "false",
80
+ "exclude_controllers" => "true",
81
+ "exclude_helpers" => "true",
82
+ "exclude_sti_subclasses" => "false",
83
+ "ignore_model_sub_dir" => "false",
84
+ "ignore_columns" => nil,
85
+ "ignore_routes" => nil,
86
+ "ignore_unknown_options" => "false",
87
+ "hide_limit_column_types" => "integer,bigint,boolean",
88
+ "hide_default_column_types" => "json,jsonb,hstore",
89
+ "skip_on_db_migrate" => "false",
90
+ "format_bare" => "true",
91
+ "format_rdoc" => "false",
92
+ "format_yard" => "false",
93
+ "format_markdown" => "false",
94
+ "sort" => "false",
95
+ "force" => "false",
96
+ "frozen" => "false",
97
+ "classified_sort" => "true",
98
+ "trace" => "false",
99
+ "wrapper_open" => nil,
100
+ "wrapper_close" => nil,
101
+ "with_comment" => "true"
102
+ )
103
+ end
104
+
105
+ # Load only essential model annotation tasks
106
+ desc "Add schema information (as comments) to model files"
107
+ task annotate_models: :environment do
108
+ puts "Running annotate for ragdoll-core models..."
109
+
110
+ # Use the CLI approach since the programmatic API doesn't respect model_dir properly
111
+ success = system("MODEL_DIR=lib/ragdoll/core/models bundle exec annotate --models --position-in-class=before --show-foreign-keys --show-indexes")
112
+
113
+ if success
114
+ puts "✅ Model annotations updated successfully!"
115
+ else
116
+ puts "⚠️ Annotate completed with warnings (database connection issues)"
117
+ end
118
+ end
119
+
120
+ desc "Remove schema information from model files"
121
+ task :remove_annotation do
122
+ AnnotateModels.remove_annotations({
123
+ "models" => "true",
124
+ "model_dir" => "lib/ragdoll/core/models"
125
+ })
126
+ end
data/lib/tasks/db.rake ADDED
@@ -0,0 +1,338 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "rake"
4
+
5
+ namespace :db do
6
+ desc "Create the database"
7
+ task :create do
8
+ require_relative "../ragdoll-core"
9
+
10
+ config = Ragdoll::Core.configuration
11
+ puts "Creating database with config: #{config.database_config.inspect}"
12
+
13
+ case config.database_config[:adapter]
14
+ when "postgresql"
15
+ puts "PostgreSQL database setup - running as superuser to create database and role..."
16
+
17
+ # Connect as superuser to create database and role
18
+ ActiveRecord::Base.establish_connection(
19
+ adapter: 'postgresql',
20
+ database: 'postgres', # Connect to postgres database initially
21
+ username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
22
+ password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
23
+ host: config.database_config[:host] || 'localhost',
24
+ port: config.database_config[:port] || 5432
25
+ )
26
+
27
+ # Run individual SQL commands to avoid transaction block issues
28
+ begin
29
+ ActiveRecord::Base.connection.execute("DROP DATABASE IF EXISTS ragdoll_development")
30
+ rescue => e
31
+ puts "Note: #{e.message}" if e.message.include?("does not exist")
32
+ end
33
+
34
+ begin
35
+ ActiveRecord::Base.connection.execute("DROP ROLE IF EXISTS ragdoll")
36
+ rescue => e
37
+ puts "Note: #{e.message}" if e.message.include?("does not exist")
38
+ end
39
+
40
+ begin
41
+ ActiveRecord::Base.connection.execute("CREATE ROLE ragdoll WITH LOGIN CREATEDB")
42
+ rescue => e
43
+ puts "Note: Role already exists, continuing..." if e.message.include?("already exists")
44
+ end
45
+
46
+ begin
47
+ ActiveRecord::Base.connection.execute <<-SQL
48
+ CREATE DATABASE ragdoll_development
49
+ WITH OWNER = ragdoll
50
+ ENCODING = 'UTF8'
51
+ CONNECTION LIMIT = -1
52
+ SQL
53
+ rescue => e
54
+ puts "Note: Database already exists, continuing..." if e.message.include?("already exists")
55
+ end
56
+
57
+ ActiveRecord::Base.connection.execute("GRANT ALL PRIVILEGES ON DATABASE ragdoll_development TO ragdoll")
58
+
59
+ # Connect to the new database to set schema privileges
60
+ ActiveRecord::Base.establish_connection(
61
+ adapter: 'postgresql',
62
+ database: 'ragdoll_development',
63
+ username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
64
+ password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
65
+ host: config.database_config[:host] || 'localhost',
66
+ port: config.database_config[:port] || 5432
67
+ )
68
+
69
+ ActiveRecord::Base.connection.execute <<-SQL
70
+ -- Grant schema privileges (must be done while connected to the database)
71
+ GRANT ALL PRIVILEGES ON SCHEMA public TO ragdoll;
72
+ GRANT CREATE ON SCHEMA public TO ragdoll;
73
+
74
+ -- Set default privileges for future objects
75
+ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO ragdoll;
76
+ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO ragdoll;
77
+ ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO ragdoll;
78
+
79
+ -- Enable pgvector extension
80
+ CREATE EXTENSION IF NOT EXISTS vector;
81
+ SQL
82
+
83
+ puts "Database and role created successfully"
84
+ when "mysql2"
85
+ # For MySQL, we'd typically create the database here
86
+ puts "MySQL database creation - ensure the database exists on your server"
87
+ end
88
+
89
+ puts "Database creation completed"
90
+ end
91
+
92
+ desc "Drop the database"
93
+ task :drop do
94
+ require_relative "../ragdoll-core"
95
+
96
+ config = Ragdoll::Core.configuration
97
+ puts "Dropping database with config: #{config.database_config.inspect}"
98
+
99
+ case config.database_config[:adapter]
100
+ when "postgresql", "mysql2"
101
+ puts "For #{config.database_config[:adapter]}, please drop the database manually on your server"
102
+ end
103
+
104
+ puts "Database drop completed"
105
+ end
106
+
107
+ desc "Setup the database (create and migrate)"
108
+ task setup: %i[create migrate]
109
+
110
+ desc "Reset the database (drop, create, and migrate)"
111
+ task reset: %i[drop create migrate]
112
+
113
+ desc "Run pending migrations"
114
+ task :migrate do
115
+ require_relative "../ragdoll-core"
116
+
117
+ puts "Running migrations..."
118
+ Ragdoll::Core::Database.setup({
119
+ auto_migrate: false
120
+ })
121
+
122
+ Ragdoll::Core::Database.migrate!
123
+ puts "Migrations completed"
124
+ end
125
+
126
+ desc "Rollback the database by one migration"
127
+ task :rollback do
128
+ require_relative "../ragdoll-core"
129
+
130
+ puts "Rolling back migrations..."
131
+ # For now, we'll implement a simple reset since our manual migration doesn't support rollback
132
+ puts "Note: Rollback not yet implemented, use db:reset to start over"
133
+ end
134
+
135
+ desc "Show migration status"
136
+ task :migrate_status do
137
+ require_relative "../ragdoll-core"
138
+
139
+ Ragdoll::Core::Database.setup({
140
+ auto_migrate: false
141
+ })
142
+
143
+ puts "\nMigration Status:"
144
+ puts "=================="
145
+
146
+ # Get migration files
147
+ migration_paths = [File.join(File.dirname(__FILE__), "..", "..", "db", "migrate")]
148
+ migration_files = Dir[File.join(migration_paths.first, "*.rb")].sort
149
+
150
+ # Get applied migrations
151
+ applied_versions = []
152
+ if ActiveRecord::Base.connection.table_exists?("schema_migrations")
153
+ applied_versions = ActiveRecord::Base.connection.select_values(
154
+ "SELECT version FROM schema_migrations ORDER BY version"
155
+ )
156
+ end
157
+
158
+ puts format("%-8s %-20s %s", "Status", "Migration ID", "Migration Name")
159
+ puts "-" * 60
160
+
161
+ migration_files.each do |migration_file|
162
+ version = File.basename(migration_file, ".rb").split("_").first
163
+ name = File.basename(migration_file, ".rb").split("_")[1..].join("_")
164
+ status = applied_versions.include?(version) ? "up" : "down"
165
+
166
+ puts format("%-8s %-20s %s", status, version, name)
167
+ end
168
+
169
+ puts "\nTotal migrations: #{migration_files.length}"
170
+ puts "Applied migrations: #{applied_versions.length}"
171
+ puts "Pending migrations: #{migration_files.length - applied_versions.length}"
172
+ end
173
+
174
+ desc "Show database schema information"
175
+ task :schema do
176
+ require_relative "../ragdoll-core"
177
+
178
+ Ragdoll::Core::Database.setup({
179
+ auto_migrate: false
180
+ })
181
+
182
+ puts "\nDatabase Schema:"
183
+ puts "================"
184
+ puts "Adapter: #{ActiveRecord::Base.connection.adapter_name}"
185
+
186
+ if ActiveRecord::Base.connection.tables.any?
187
+ ActiveRecord::Base.connection.tables.sort.each do |table|
188
+ puts "\nTable: #{table}"
189
+ columns = ActiveRecord::Base.connection.columns(table)
190
+ columns.each do |column|
191
+ puts " #{column.name}: #{column.type} (#{column.sql_type})#{unless column.null
192
+ ' NOT NULL'
193
+ end}#{if column.default
194
+ " DEFAULT #{column.default.inspect}"
195
+ end}"
196
+ end
197
+
198
+ # Show indexes
199
+ indexes = ActiveRecord::Base.connection.indexes(table)
200
+ next unless indexes.any?
201
+
202
+ puts " Indexes:"
203
+ indexes.each do |index|
204
+ unique_text = index.unique ? " (unique)" : ""
205
+ puts " #{index.name}: [#{index.columns.join(', ')}]#{unique_text}"
206
+ end
207
+ end
208
+ else
209
+ puts "No tables found. Run 'rake db:migrate' to create tables."
210
+ end
211
+ end
212
+
213
+ desc "Open database console"
214
+ task :console do
215
+ require_relative "../ragdoll-core"
216
+
217
+ config = Ragdoll::Core.configuration
218
+
219
+ case config.database_config[:adapter]
220
+ when "postgresql"
221
+ db_config = config.database_config
222
+ psql_cmd = "psql"
223
+ psql_cmd += " -h #{db_config[:host]}" if db_config[:host]
224
+ psql_cmd += " -p #{db_config[:port]}" if db_config[:port]
225
+ psql_cmd += " -U #{db_config[:username]}" if db_config[:username]
226
+ psql_cmd += " #{db_config[:database]}"
227
+ puts "Opening PostgreSQL console..."
228
+ system(psql_cmd)
229
+ when "mysql2"
230
+ db_config = config.database_config
231
+ mysql_cmd = "mysql"
232
+ mysql_cmd += " -h #{db_config[:host]}" if db_config[:host]
233
+ mysql_cmd += " -P #{db_config[:port]}" if db_config[:port]
234
+ mysql_cmd += " -u #{db_config[:username]}" if db_config[:username]
235
+ mysql_cmd += " -p" if db_config[:password]
236
+ mysql_cmd += " #{db_config[:database]}"
237
+ puts "Opening MySQL console..."
238
+ system(mysql_cmd)
239
+ else
240
+ puts "Console not supported for adapter: #{config.database_config[:adapter]}"
241
+ end
242
+ end
243
+
244
+ desc "Show database statistics"
245
+ task :stats do
246
+ require_relative "../ragdoll-core"
247
+
248
+ Ragdoll::Core::Database.setup({
249
+ auto_migrate: false
250
+ })
251
+
252
+ puts "\nDatabase Statistics:"
253
+ puts "==================="
254
+
255
+ if ActiveRecord::Base.connection.table_exists?("ragdoll_documents")
256
+ doc_count = ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM ragdoll_documents")
257
+ puts "Documents: #{doc_count}"
258
+
259
+ if doc_count.positive?
260
+ doc_types = ActiveRecord::Base.connection.select_rows(
261
+ "SELECT document_type, COUNT(*) FROM ragdoll_documents GROUP BY document_type"
262
+ )
263
+ puts "Document types:"
264
+ doc_types.each { |type, count| puts " #{type}: #{count}" }
265
+
266
+ statuses = ActiveRecord::Base.connection.select_rows(
267
+ "SELECT status, COUNT(*) FROM ragdoll_documents GROUP BY status"
268
+ )
269
+ puts "Document statuses:"
270
+ statuses.each { |status, count| puts " #{status}: #{count}" }
271
+ end
272
+ else
273
+ puts "Documents table does not exist"
274
+ end
275
+
276
+ if ActiveRecord::Base.connection.table_exists?("ragdoll_embeddings")
277
+ embedding_count = ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM ragdoll_embeddings")
278
+ puts "Embeddings: #{embedding_count}"
279
+
280
+ if embedding_count.positive?
281
+ models = ActiveRecord::Base.connection.select_rows(
282
+ "SELECT model_name, COUNT(*) FROM ragdoll_embeddings GROUP BY model_name"
283
+ )
284
+ puts "Embedding models:"
285
+ models.each { |model, count| puts " #{model}: #{count}" }
286
+
287
+ usage_stats = ActiveRecord::Base.connection.select_one(
288
+ "SELECT AVG(usage_count) as avg_usage, MAX(usage_count) as max_usage FROM ragdoll_embeddings"
289
+ )
290
+ puts "Usage statistics:"
291
+ puts " Average usage: #{usage_stats['avg_usage'].to_f.round(2)}"
292
+ puts " Max usage: #{usage_stats['max_usage']}"
293
+ end
294
+ else
295
+ puts "Embeddings table does not exist"
296
+ end
297
+ end
298
+
299
+ desc "Truncate all tables (remove all data but keep structure)"
300
+ task :truncate do
301
+ require_relative "../ragdoll-core"
302
+
303
+ Ragdoll::Core::Database.setup({
304
+ auto_migrate: false
305
+ })
306
+
307
+ puts "Truncating all tables..."
308
+
309
+ # Disable foreign key checks temporarily
310
+ case ActiveRecord::Base.connection.adapter_name.downcase
311
+ when "postgresql"
312
+ ActiveRecord::Base.connection.execute("SET session_replication_role = 'replica'")
313
+ when "mysql"
314
+ ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS = 0")
315
+ end
316
+
317
+ # Truncate tables in correct order (dependent tables first)
318
+ %w[ragdoll_embeddings ragdoll_documents].each do |table|
319
+ if ActiveRecord::Base.connection.table_exists?(table)
320
+ ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
321
+ puts "Truncated #{table}"
322
+ end
323
+ end
324
+
325
+ # Re-enable foreign key checks
326
+ case ActiveRecord::Base.connection.adapter_name.downcase
327
+ when "postgresql"
328
+ ActiveRecord::Base.connection.execute("SET session_replication_role = 'origin'")
329
+ when "mysql"
330
+ ActiveRecord::Base.connection.execute("SET FOREIGN_KEY_CHECKS = 1")
331
+ end
332
+
333
+ puts "All tables truncated"
334
+ end
335
+ end
336
+
337
+ # Make db tasks available as top-level commands
338
+ task db: "db:migrate_status"
metadata CHANGED
@@ -1,28 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ragdoll
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dewayne VanHoozer
8
8
  bindir: bin
9
9
  cert_chain: []
10
- date: 2025-02-19 00:00:00.000000000 Z
11
- dependencies:
12
- - !ruby/object:Gem::Dependency
13
- name: rails
14
- requirement: !ruby/object:Gem::Requirement
15
- requirements:
16
- - - "~>"
17
- - !ruby/object:Gem::Version
18
- version: '7.1'
19
- type: :runtime
20
- prerelease: false
21
- version_requirements: !ruby/object:Gem::Requirement
22
- requirements:
23
- - - "~>"
24
- - !ruby/object:Gem::Version
25
- version: '7.1'
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies: []
26
12
  description: Under development. Contributors welcome.
27
13
  email:
28
14
  - dvanhoozer@gmail.com
@@ -32,46 +18,67 @@ extra_rdoc_files: []
32
18
  files:
33
19
  - README.md
34
20
  - Rakefile
21
+ - app/jobs/ragdoll/extract_keywords_job.rb
22
+ - app/jobs/ragdoll/extract_text_job.rb
23
+ - app/jobs/ragdoll/generate_embeddings_job.rb
24
+ - app/jobs/ragdoll/generate_summary_job.rb
25
+ - app/lib/ragdoll/metadata_schemas.rb
26
+ - app/models/ragdoll/audio_content.rb
27
+ - app/models/ragdoll/content.rb
35
28
  - app/models/ragdoll/document.rb
36
29
  - app/models/ragdoll/embedding.rb
37
- - config/initializers/ragdoll.rb
38
- - config/routes.rb
39
- - db/migrate/20250218123456_create_documents.rb
40
- - lib/config/database.yml
41
- - lib/config/ragdoll.yml
30
+ - app/models/ragdoll/image_content.rb
31
+ - app/models/ragdoll/text_content.rb
32
+ - app/services/ragdoll/configuration_service.rb
33
+ - app/services/ragdoll/document_management.rb
34
+ - app/services/ragdoll/document_processor.rb
35
+ - app/services/ragdoll/embedding_service.rb
36
+ - app/services/ragdoll/image_description_service.rb
37
+ - app/services/ragdoll/metadata_generator.rb
38
+ - app/services/ragdoll/model_resolver.rb
39
+ - app/services/ragdoll/search_engine.rb
40
+ - app/services/ragdoll/text_chunker.rb
41
+ - app/services/ragdoll/text_generation_service.rb
42
+ - db/migrate/001_enable_postgresql_extensions.rb
43
+ - db/migrate/004_create_ragdoll_documents.rb
44
+ - db/migrate/005_create_ragdoll_embeddings.rb
45
+ - db/migrate/006_create_ragdoll_contents.rb
46
+ - lib/ragdoll-core.rb
42
47
  - lib/ragdoll.rb
43
- - lib/ragdoll/engine.rb
44
- - lib/ragdoll/import_job.rb
45
- - lib/ragdoll/ingestion.rb
46
- - lib/ragdoll/search.rb
47
- - lib/ragdoll/version.rb
48
- - lib/tasks/import_task.thor
49
- - lib/tasks/jobs_task.thor
50
- - lib/tasks/ragdoll_tasks.thor
51
- - lib/tasks/search_task.thor
48
+ - lib/ragdoll/core.rb
49
+ - lib/ragdoll/core/client.rb
50
+ - lib/ragdoll/core/configuration.rb
51
+ - lib/ragdoll/core/database.rb
52
+ - lib/ragdoll/core/errors.rb
53
+ - lib/ragdoll/core/model.rb
54
+ - lib/ragdoll/core/shrine_config.rb
55
+ - lib/ragdoll/core/version.rb
56
+ - lib/tasks/annotate.rake
57
+ - lib/tasks/db.rake
52
58
  homepage: https://github.com/MadBomber/ragdoll
53
59
  licenses:
54
60
  - MIT
55
61
  metadata:
56
62
  allowed_push_host: https://rubygems.org
57
63
  homepage_uri: https://github.com/MadBomber/ragdoll
58
- source_code_uri: https://github.com/MadBomber/ragdoll
64
+ source_code_uri: https://github.com/MadBomber/ragdoll/blob/main
59
65
  changelog_uri: https://github.com/MadBomber/ragdoll/blob/main/CHANGELOG.md
60
66
  rdoc_options: []
61
67
  require_paths:
62
68
  - lib
69
+ - app/models
63
70
  required_ruby_version: !ruby/object:Gem::Requirement
64
71
  requirements:
65
72
  - - ">="
66
73
  - !ruby/object:Gem::Version
67
- version: 3.1.0
74
+ version: 3.2.0
68
75
  required_rubygems_version: !ruby/object:Gem::Requirement
69
76
  requirements:
70
77
  - - ">="
71
78
  - !ruby/object:Gem::Version
72
79
  version: '0'
73
80
  requirements: []
74
- rubygems_version: 3.6.3
81
+ rubygems_version: 3.7.1
75
82
  specification_version: 4
76
- summary: Ruby on Rails Engine
83
+ summary: Multi-Modal Retrieval Augmented Generation
77
84
  test_files: []
@@ -1,6 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # Initializer for Ragdoll engine
4
- Ragdoll.configure do |config|
5
- # Set configuration options here
6
- end
data/config/routes.rb DELETED
@@ -1,5 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- Ragdoll::Engine.routes.draw do
4
- # Define your engine routes here
5
- end
@@ -1,20 +0,0 @@
1
- # This migration creates the documents table with necessary extensions for PostgreSQL.
2
-
3
- module Ragdoll
4
- class CreateDocuments < ActiveRecord::Migration[7.0]
5
- def change
6
- enable_extension 'pg_trgm'
7
- enable_extension 'fuzzystrmatch'
8
-
9
- create_table :documents do |t|
10
- t.string :location
11
- t.string :summary
12
- t.string :type
13
- t.datetime :processing_started_at
14
- t.datetime :processing_finished_at
15
-
16
- t.timestamps
17
- end
18
- end
19
- end
20
- end
@@ -1,28 +0,0 @@
1
- # This file contains the database configuration for the Ragdoll gem, using environment variables.
2
-
3
- default: &default
4
- adapter: postgresql
5
- encoding: unicode
6
- pool: <%= ENV.fetch("RAGDOLL_POOL", 5) %>
7
- timeout: <%= ENV.fetch("RAGDOLL_TIMEOUT", 5000) %>
8
-
9
- development:
10
- <<: *default
11
- host: <%= ENV.fetch("RAGDOLL_HOST", "localhost") %>
12
- database: <%= ENV.fetch("RAGDOLL_DATABASE", "ragdoll_development") %>
13
- username: <%= ENV.fetch("RAGDOLL_USER", "user") %>
14
- password: <%= ENV.fetch("RAGDOLL_PASSWORD", "password") %>
15
-
16
- test:
17
- <<: *default
18
- host: <%= ENV.fetch("RAGDOLL_HOST", "localhost") %>
19
- database: <%= ENV.fetch("RAGDOLL_DATABASE", "ragdoll_test") %>
20
- username: <%= ENV.fetch("RAGDOLL_USER", "user") %>
21
- password: <%= ENV.fetch("RAGDOLL_PASSWORD", "password") %>
22
-
23
- production:
24
- <<: *default
25
- host: <%= ENV.fetch("RAGDOLL_HOST") %>
26
- database: <%= ENV.fetch("RAGDOLL_DATABASE") %>
27
- username: <%= ENV.fetch("RAGDOLL_USER") %>
28
- password: <%= ENV.fetch("RAGDOLL_PASSWORD") %>
@@ -1,31 +0,0 @@
1
- # This file contains the default configuration settings for the Ragdoll gem, including database configurations.
2
-
3
- default: &default
4
- database:
5
- host: localhost
6
- database: ragdoll_development
7
- user: user
8
- password: password
9
- pool: 5
10
- timeout: 5000
11
-
12
- llm:
13
- embeddings_model: "llama-2-7b"
14
- reranking_model: "llama-2-13b"
15
- chat_model: "llama-2-70b"
16
-
17
- development:
18
- <<: *default
19
-
20
- test:
21
- <<: *default
22
- database:
23
- database: ragdoll_test
24
-
25
- production:
26
- <<: *default
27
- database:
28
- host: <%= ENV.fetch("RAGDOLL_HOST") %>
29
- database: <%= ENV.fetch("RAGDOLL_DATABASE") %>
30
- user: <%= ENV.fetch("RAGDOLL_USER") %>
31
- password: <%= ENV.fetch("RAGDOLL_PASSWORD") %>