ragdoll 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +68 -4
- data/README.md +86 -1
- data/Rakefile +4 -2
- data/app/models/ragdoll/document.rb +115 -12
- data/app/models/ragdoll/embedding.rb +36 -4
- data/app/models/ragdoll/search.rb +1 -1
- data/app/services/ragdoll/document_management.rb +117 -9
- data/app/services/ragdoll/document_processor.rb +67 -31
- data/app/services/ragdoll/search_engine.rb +13 -2
- data/db/migrate/{001_enable_postgresql_extensions.rb → 20250815234901_enable_postgresql_extensions.rb} +7 -8
- data/db/migrate/20250815234902_create_ragdoll_documents.rb +117 -0
- data/db/migrate/{005_create_ragdoll_embeddings.rb → 20250815234903_create_ragdoll_embeddings.rb} +13 -10
- data/db/migrate/{006_create_ragdoll_contents.rb → 20250815234904_create_ragdoll_contents.rb} +14 -11
- data/db/migrate/{007_create_ragdoll_searches.rb → 20250815234905_create_ragdoll_searches.rb} +24 -20
- data/db/migrate/{008_create_ragdoll_search_results.rb → 20250815234906_create_ragdoll_search_results.rb} +16 -16
- data/lib/ragdoll/core/client.rb +2 -2
- data/lib/ragdoll/core/database.rb +8 -3
- data/lib/ragdoll/core/version.rb +1 -1
- data/lib/tasks/db.rake +63 -15
- metadata +7 -7
- data/db/migrate/004_create_ragdoll_documents.rb +0 -70
data/lib/tasks/db.rake
CHANGED
@@ -25,22 +25,17 @@ namespace :db do
|
|
25
25
|
)
|
26
26
|
|
27
27
|
# Run individual SQL commands to avoid transaction block issues
|
28
|
-
|
29
|
-
ActiveRecord::Base.connection.execute("DROP DATABASE IF EXISTS ragdoll_development")
|
30
|
-
rescue => e
|
31
|
-
puts "Note: #{e.message}" if e.message.include?("does not exist")
|
32
|
-
end
|
33
|
-
|
34
|
-
begin
|
35
|
-
ActiveRecord::Base.connection.execute("DROP ROLE IF EXISTS ragdoll")
|
36
|
-
rescue => e
|
37
|
-
puts "Note: #{e.message}" if e.message.include?("does not exist")
|
38
|
-
end
|
28
|
+
# Note: Removed the DROP DATABASE/ROLE here since that should be done via db:drop task
|
39
29
|
|
40
30
|
begin
|
41
31
|
ActiveRecord::Base.connection.execute("CREATE ROLE ragdoll WITH LOGIN CREATEDB")
|
32
|
+
puts "Role 'ragdoll' created successfully"
|
42
33
|
rescue => e
|
43
|
-
|
34
|
+
if e.message.include?("already exists")
|
35
|
+
puts "Note: Role 'ragdoll' already exists, continuing..."
|
36
|
+
else
|
37
|
+
raise e
|
38
|
+
end
|
44
39
|
end
|
45
40
|
|
46
41
|
begin
|
@@ -50,8 +45,16 @@ namespace :db do
|
|
50
45
|
ENCODING = 'UTF8'
|
51
46
|
CONNECTION LIMIT = -1
|
52
47
|
SQL
|
48
|
+
puts "Database 'ragdoll_development' created successfully"
|
53
49
|
rescue => e
|
54
|
-
|
50
|
+
if e.message.include?("already exists")
|
51
|
+
puts "ERROR: Database 'ragdoll_development' already exists!"
|
52
|
+
puts "Please run 'rake db:drop' first to remove the existing database, then run 'rake db:create' again."
|
53
|
+
puts "Or use 'rake db:reset' to drop, create, and migrate in one step."
|
54
|
+
exit 1
|
55
|
+
else
|
56
|
+
raise e
|
57
|
+
end
|
55
58
|
end
|
56
59
|
|
57
60
|
ActiveRecord::Base.connection.execute("GRANT ALL PRIVILEGES ON DATABASE ragdoll_development TO ragdoll")
|
@@ -97,8 +100,53 @@ namespace :db do
|
|
97
100
|
puts "Dropping database with config: #{config.database.inspect}"
|
98
101
|
|
99
102
|
case config.database[:adapter]
|
100
|
-
when "postgresql"
|
101
|
-
puts "
|
103
|
+
when "postgresql"
|
104
|
+
puts "PostgreSQL database drop - running as superuser to drop database and role..."
|
105
|
+
|
106
|
+
# Connect as superuser to drop database and role
|
107
|
+
ActiveRecord::Base.establish_connection(
|
108
|
+
adapter: 'postgresql',
|
109
|
+
database: 'postgres', # Connect to postgres database initially
|
110
|
+
username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
|
111
|
+
password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
|
112
|
+
host: config.database[:host] || 'localhost',
|
113
|
+
port: config.database[:port] || 5432
|
114
|
+
)
|
115
|
+
|
116
|
+
# Drop the database if it exists
|
117
|
+
begin
|
118
|
+
ActiveRecord::Base.connection.execute("DROP DATABASE IF EXISTS ragdoll_development")
|
119
|
+
puts "Database 'ragdoll_development' dropped successfully"
|
120
|
+
rescue => e
|
121
|
+
puts "Error dropping database: #{e.message}"
|
122
|
+
end
|
123
|
+
|
124
|
+
# Optionally drop the role (commented out by default to preserve user)
|
125
|
+
# begin
|
126
|
+
# ActiveRecord::Base.connection.execute("DROP ROLE IF EXISTS ragdoll")
|
127
|
+
# puts "Role 'ragdoll' dropped successfully"
|
128
|
+
# rescue => e
|
129
|
+
# puts "Error dropping role: #{e.message}"
|
130
|
+
# end
|
131
|
+
|
132
|
+
when "mysql2"
|
133
|
+
puts "MySQL database drop - connecting to drop database..."
|
134
|
+
|
135
|
+
# Connect without specifying database
|
136
|
+
ActiveRecord::Base.establish_connection(
|
137
|
+
adapter: 'mysql2',
|
138
|
+
username: config.database[:username],
|
139
|
+
password: config.database[:password],
|
140
|
+
host: config.database[:host] || 'localhost',
|
141
|
+
port: config.database[:port] || 3306
|
142
|
+
)
|
143
|
+
|
144
|
+
begin
|
145
|
+
ActiveRecord::Base.connection.execute("DROP DATABASE IF EXISTS #{config.database[:database]}")
|
146
|
+
puts "Database '#{config.database[:database]}' dropped successfully"
|
147
|
+
rescue => e
|
148
|
+
puts "Error dropping database: #{e.message}"
|
149
|
+
end
|
102
150
|
end
|
103
151
|
|
104
152
|
puts "Database drop completed"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ragdoll
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dewayne VanHoozer
|
@@ -406,12 +406,12 @@ files:
|
|
406
406
|
- app/services/ragdoll/search_engine.rb
|
407
407
|
- app/services/ragdoll/text_chunker.rb
|
408
408
|
- app/services/ragdoll/text_generation_service.rb
|
409
|
-
- db/migrate/
|
410
|
-
- db/migrate/
|
411
|
-
- db/migrate/
|
412
|
-
- db/migrate/
|
413
|
-
- db/migrate/
|
414
|
-
- db/migrate/
|
409
|
+
- db/migrate/20250815234901_enable_postgresql_extensions.rb
|
410
|
+
- db/migrate/20250815234902_create_ragdoll_documents.rb
|
411
|
+
- db/migrate/20250815234903_create_ragdoll_embeddings.rb
|
412
|
+
- db/migrate/20250815234904_create_ragdoll_contents.rb
|
413
|
+
- db/migrate/20250815234905_create_ragdoll_searches.rb
|
414
|
+
- db/migrate/20250815234906_create_ragdoll_search_results.rb
|
415
415
|
- lib/ragdoll-core.rb
|
416
416
|
- lib/ragdoll.rb
|
417
417
|
- lib/ragdoll/core.rb
|
@@ -1,70 +0,0 @@
|
|
1
|
-
class CreateRagdollDocuments < ActiveRecord::Migration[7.0]
|
2
|
-
def change
|
3
|
-
create_table :ragdoll_documents,
|
4
|
-
comment: "Core documents table with LLM-generated structured metadata" do |t|
|
5
|
-
|
6
|
-
t.string :location, null: false,
|
7
|
-
comment: "Source location of document (file path, URL, or identifier)"
|
8
|
-
|
9
|
-
t.string :title, null: false,
|
10
|
-
comment: "Human-readable document title for display and search"
|
11
|
-
|
12
|
-
t.text :summary, null: false, default: "",
|
13
|
-
comment: "LLM-generated summary of document content"
|
14
|
-
|
15
|
-
t.text :keywords , null: false, default: "",
|
16
|
-
comment: "LLM-generated comma-separated keywords of document"
|
17
|
-
|
18
|
-
t.string :document_type, null: false, default: "text",
|
19
|
-
comment: "Document format type"
|
20
|
-
|
21
|
-
t.string :status, null: false, default: "pending",
|
22
|
-
comment: "Document processing status"
|
23
|
-
|
24
|
-
t.json :metadata, default: {},
|
25
|
-
comment: "LLM-generated structured metadata about the file"
|
26
|
-
|
27
|
-
t.timestamp :file_modified_at, null: false, default: -> { "CURRENT_TIMESTAMP" },
|
28
|
-
comment: "Timestamp when the source file was last modified"
|
29
|
-
|
30
|
-
t.timestamps null: false,
|
31
|
-
comment: "Standard creation and update timestamps"
|
32
|
-
|
33
|
-
###########
|
34
|
-
# Indexes #
|
35
|
-
###########
|
36
|
-
|
37
|
-
t.index :location, unique: true,
|
38
|
-
comment: "Unique index for document source lookup"
|
39
|
-
|
40
|
-
t.index :title,
|
41
|
-
comment: "Index for title-based search"
|
42
|
-
|
43
|
-
t.index :document_type,
|
44
|
-
comment: "Index for filtering by document type"
|
45
|
-
|
46
|
-
t.index :status,
|
47
|
-
comment: "Index for filtering by processing status"
|
48
|
-
|
49
|
-
t.index :created_at,
|
50
|
-
comment: "Index for chronological sorting"
|
51
|
-
|
52
|
-
t.index %i[document_type status],
|
53
|
-
comment: "Composite index for type+status filtering"
|
54
|
-
|
55
|
-
t.index "to_tsvector('english', COALESCE(title, '') ||
|
56
|
-
' ' ||
|
57
|
-
COALESCE(metadata->>'summary', '') ||
|
58
|
-
' ' || COALESCE(metadata->>'keywords', '') ||
|
59
|
-
' ' || COALESCE(metadata->>'description', ''))",
|
60
|
-
using: :gin, name: "index_ragdoll_documents_on_fulltext_search",
|
61
|
-
comment: "Full-text search across title and metadata fields"
|
62
|
-
|
63
|
-
t.index "(metadata->>'document_type')", name: "index_ragdoll_documents_on_metadata_type",
|
64
|
-
comment: "Index for filtering by document type"
|
65
|
-
|
66
|
-
t.index "(metadata->>'classification')", name: "index_ragdoll_documents_on_metadata_classification",
|
67
|
-
comment: "Index for filtering by document classification"
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|