smart_rag 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19e9abcf3817e98fad15499879a25db6fbb563bc189f5a8f0c58fbea2145cbf8
-  data.tar.gz: 88f7e4a0f1929b83c9346e8261928bdd72690de4c9da3689a3ca7882a548a6b1
+  metadata.gz: db8be81c26762690c4b8b4d2a3bb7c9b3081fd94ce9a05e379964df92ba78b22
+  data.tar.gz: 47465a464dff0874022bfcfd37b7eb1115291f1af71b6d4afeb5eefc6eaf249e
 SHA512:
-  metadata.gz: f3c7c945032e61c816f383ea6ae1ea87b0be9d4fb2a794175543489e91763825f73fbb925c2a52b37d786af2e1dd126180b4da3cb5c31ded4130e1fa5bc4ebda
-  data.tar.gz: 6de0148938c32c3842d3d3e2a8f31641c508be193d70e9fa11d01c93dd413e3e805f2fec3afb348981455566989654436cb5e7f70c0351b2b94e1dc21978a94a
+  metadata.gz: 68a9f3d4fffa35954489285189d49ed73cab62acd304fd441249aeffa3b454fda816f4799cfd111e2d2a4392d18f2aba4897563201c555b91c70ba7c6d25973e
+  data.tar.gz: c3f5efc011510a414aaae99f956119f4f3446391c6352567502e4572e0eeb5df0ac5690c02f50c259293bb55733f59f492b34b5c2e3ee0a31421a223d683f7ff

data/.env.example

@@ -0,0 +1,252 @@
+# SmartRAG Environment Configuration
+# Copy this file to .env and customize for your environment
+
+# =
+# PostgreSQL Database Configuration
+# =
+
+# Development database (default)
+SMARTRAG_DB_HOST=
+SMARTRAG_DB_PORT=
+SMARTRAG_DB_NAME=
+SMARTRAG_DB_USER=
+SMARTRAG_DB_PASSWORD=
+
+# Test database (for RSpec tests)
+SMARTRAG_TEST_DB_HOST=
+SMARTRAG_TEST_DB_PORT=
+SMARTRAG_TEST_DB_NAME=
+SMARTRAG_TEST_DB_USER=
+SMARTRAG_TEST_DB_PASSWORD=
+
+# =
+# API Configuration
+# =
+
+# Optional: OpenAI API key (for LLM if you use OpenAI)
+OPENAI_API_KEY=
+
+# Media ingestion and built-in semantic extraction
+MEDIA_MAX_FILE_SIZE_MB=50
+MEDIA_MAX_DURATION_MS=7200000
+MEDIA_COMMAND_TIMEOUT=120
+MEDIA_ALLOW_PRIVATE_URLS=false
+MEDIA_ALLOWED_HOSTS=
+MEDIA_CONTENT_STORE_ENABLED=false
+MEDIA_CONTENT_STORE_PROVIDER=local
+MEDIA_CONTENT_STORE_ROOT=
+MEDIA_S3_BUCKET=
+MEDIA_S3_REGION=us-east-1
+MEDIA_S3_PREFIX=smart-rag/media
+MEDIA_S3_ENDPOINT=
+MEDIA_S3_FORCE_PATH_STYLE=false
+MEDIA_S3_ACCESS_KEY_ID=
+MEDIA_S3_SECRET_ACCESS_KEY=
+MEDIA_OPENAI_BASE_URL=https://api.openai.com
+MEDIA_OPENAI_API_KEY=
+MEDIA_VISION_MODEL=
+MEDIA_TRANSCRIPTION_MODEL=
+MEDIA_TRANSCRIPTION_LANGUAGE=
+MEDIA_OCR_PROVIDER=
+MEDIA_OCR_LANGUAGE=
+MEDIA_JOB_MAX_ATTEMPTS=3
+MEDIA_JOB_STALE_AFTER_SECONDS=900
+MEDIA_JOB_RETENTION_SECONDS=604800
+MEDIA_JOB_HEARTBEAT_INTERVAL_SECONDS=30
+SMARTRAG_MEDIA_JOB_UPLOAD_DIR=
+
+# HTTP authentication and per-principal quotas (0 disables a quota)
+SMARTRAG_HTTP_AUTH_ENABLED=false
+SMARTRAG_HTTP_PRINCIPAL=default
+SMARTRAG_HTTP_TOKEN=
+SMARTRAG_REQUESTS_PER_MINUTE=0
+SMARTRAG_UPLOAD_BYTES_PER_DAY=0
+
+# Embedding API configuration (default: local Ollama qwen3-embedding)
+EMBEDDING_PROVIDER=local
+EMBEDDING_API_KEY=ollama-local
+EMBEDDING_ENDPOINT=http://localhost:11434/v1/embeddings
+EMBEDDING_MODEL=qwen3-embedding
+EMBEDDING_DIMENSIONS=4096
+OLLAMA_BASE_URL=http://localhost:11434/v1/
+OLLAMA_API_KEY=ollama-local
+
+# LLM Configuration
+LLM_PROVIDER=openai
+LLM_API_KEY=ollama-local
+LLM_ENDPOINT=http://localhost:11434/v1/chat/completions
+LLM_MODEL=qwen3
+SILICONFLOW_API_KEY=
+
+# =
+# Search Configuration
+# =
+
+# Default language for search (en, zh, ja, ko)
+DEFAULT_LANGUAGE=
+
+# Maximum search results
+MAX_SEARCH_RESULTS=
+
+# Hybrid search weights (0.0 to 1.0)
+FULLTEXT_WEIGHT=
+VECTOR_WEIGHT=
+
+# RRF (Reciprocal Rank Fusion) parameter
+RRF_K=
+
+# Search result limits
+DEFAULT_SEARCH_LIMIT=
+
+# Tag boost weight for vector search (0.0 to 1.0)
+TAG_BOOST_WEIGHT=
+
+# Maximum distance threshold for relevant results (0.0 to 1.0)
+MAX_DISTANCE_THRESHOLD=
+
+# =
+# Chinese Text Processing (pg_jieba)
+# =
+
+# Enable Chinese text segmentation
+ENABLE_JIEBA=
+
+# Optional: Path to custom dictionary
+# JIEBA_DICT_PATH=
+
+# =
+# Document Chunking Configuration
+# =
+
+# Maximum characters per chunk
+CHUNK_MAX_CHARS=
+
+# Overlap between chunks (preserve context)
+CHUNK_OVERLAP=
+
+# Split by Markdown headers first
+SPLIT_BY_HEADERS=
+
+# Minimum chunk size (discard smaller chunks)
+MIN_CHUNK_SIZE=
+
+# =
+# Document Processing
+# =
+
+# Maximum file size in MB
+MAX_FILE_SIZE=
+
+# Download timeout for URLs (seconds)
+DOWNLOAD_TIMEOUT=
+
+# =
+# Redis Cache Configuration (Optional)
+# =
+
+# Enable search result caching
+# CACHE_ENABLED=
+
+# Redis URL for caching
+# REDIS_URL=
+
+# Cache TTL in seconds (default: 1 hour)
+# CACHE_TTL=
+
+# =
+# Performance Settings
+# =
+
+# Thread pool size for parallel processing
+THREAD_POOL_SIZE=
+
+# Batch size for database operations
+DB_BATCH_SIZE=
+
+# Enable connection pooling
+CONNECTION_POOLING=
+
+# =
+# Logging Configuration
+# =
+
+# Log level: debug, info, warn, error, fatal
+LOG_LEVEL=
+
+# Log format: json, plain
+LOG_FORMAT=
+
+# Optional: Log file path (leave empty for stdout)
+# LOG_FILE_PATH=
+
+# Enable query logging
+ENABLE_QUERY_LOG=
+
+# =
+# LLM Generation Parameters
+# =
+
+# Temperature for LLM (0.0 to 2.0, higher =
+LLM_TEMPERATURE=
+
+# Maximum tokens to generate
+LLM_MAX_TOKENS=
+
+# LLM request timeout (seconds)
+LLM_TIMEOUT=
+
+# =
+# Monitoring and Analytics
+# =
+
+# Log slow queries (over threshold)
+LOG_SLOW_QUERIES=
+
+# Slow query threshold in milliseconds
+SLOW_QUERY_THRESHOLD=
+
+# =
+# Index Maintenance
+# =
+
+# Enable partitioned indexes
+ENABLE_PARTITION=
+
+# Enable auto vacuum
+AUTO_VACUUM=
+
+# Optional: Index rebuild schedule (cron format)
+# INDEX_REBUILD_SCHEDULE=
+
+# =
+# Azure OpenAI Configuration (if using Azure)
+# =
+
+# AZURE_OPENAI_API_KEY=
+# AZURE_OPENAI_ENDPOINT=
+# AZURE_OPENAI_DEPLOYMENT_NAME=
+# AZURE_OPENAI_API_VERSION=
+
+# =
+# Local Embedding Model Configuration
+# =
+
+# If using local embeddings (e.g., via Ollama)
+# EMBEDDING_PROVIDER=local
+# EMBEDDING_ENDPOINT=http://localhost:11434/v1/embeddings
+# EMBEDDING_MODEL=qwen3-embedding
+# EMBEDDING_DIMENSIONS=4096
+
+# =
+# Application Environment
+# =
+
+# Rack environment (development, test, production)
+RACK_ENV=
+
+# =
+# Migration Control
+# =
+
+# Set to false to skip seed data
+# SEED_DATA=

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/AGENTS.md

@@ -0,0 +1,33 @@
+# Repository Guidelines
+
+## Project Structure & Module Organization
+- `lib/` contains the SmartRAG library code. Core logic lives under `lib/smart_rag/core`, and services under `lib/smart_rag/services`.
+- `test/` holds runnable scripts (e.g., `test_rag.rb`, `import_doc.rb`) and sample markdown documents used for manual testing.
+- `spec/` contains RSpec tests (when present).
+- `config/` stores application configuration, including database and full-text search settings.
+- `db/` contains SQL scripts and database-related artifacts.
+
+## Build, Test, and Development Commands
+- `bundle install` installs Ruby dependencies.
+- `ruby test/import_doc.rb import` imports test documents into the local database.
+- `ruby test/test_rag.rb` runs the end-to-end RAG tests in `test/`.
+- `bundle exec rspec` runs any RSpec tests in `spec/`.
+
+## Coding Style & Naming Conventions
+- Ruby files use 2-space indentation.
+- Class/module names use `CamelCase`; methods and files use `snake_case`.
+- Keep public API methods in `lib/smart_rag.rb` small and delegate to `core/` or `services/`.
+
+## Testing Guidelines
+- End-to-end checks are in `test/test_rag.rb`; run after reindexing or data changes.
+- RSpec tests (when present) should live in `spec/` and follow `*_spec.rb` naming.
+- If a test depends on data, ensure `test/import_doc.rb` has been run.
+
+## Commit & Pull Request Guidelines
+- Git history shows short messages like `fix bug`; there is no strict convention yet.
+- Use concise, imperative commit messages (e.g., `fix fulltext indexing`).
+- PRs should include: a short summary, steps to verify, and any data/setup notes.
+
+## Security & Configuration Tips
+- Database credentials live in `config/smart_rag.yml` or environment variables (see `config/`).
+- Avoid committing API keys; use `.env.example` as the template.