@devilsdev/rag-pipeline-utils 2.1.7 → 2.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/LICENSE +674 -674
  2. package/README.md +395 -186
  3. package/bin/cli.js +14 -62
  4. package/contracts/embedder-contract.json +52 -0
  5. package/contracts/llm-contract.json +36 -0
  6. package/contracts/loader-contract.json +43 -0
  7. package/contracts/retriever-contract.json +68 -0
  8. package/dist/index.cjs +67 -0
  9. package/dist/index.mjs +51 -0
  10. package/package.json +108 -23
  11. package/.codecov.yml +0 -21
  12. package/.dockerignore +0 -14
  13. package/.eslintignore +0 -5
  14. package/.eslintrc.json +0 -28
  15. package/.github/PROJECT_ROADMAP.md +0 -102
  16. package/.github/PULL_REQUEST_TEMPLATE.md +0 -8
  17. package/.github/release-template.md +0 -20
  18. package/.github/workflows/auto-release-note.yml +0 -73
  19. package/.github/workflows/ci.yml +0 -55
  20. package/.github/workflows/docs-ci.yml +0 -46
  21. package/.github/workflows/docs-deploy.yml +0 -64
  22. package/.github/workflows/enforce-release-review.yml +0 -50
  23. package/.github/workflows/generate-blog-post.yml +0 -44
  24. package/.github/workflows/post-release-generate-blog.yml +0 -71
  25. package/.github/workflows/publish-release-blog.yml +0 -47
  26. package/.github/workflows/release-sync.yml +0 -50
  27. package/.github/workflows/roadmap-label-sync.yml +0 -35
  28. package/.github/workflows/roadmap-labels.yml +0 -67
  29. package/.github/workflows/roadmap-sync.yml +0 -44
  30. package/.github/workflows/sync-roadmap.yml +0 -62
  31. package/.github/workflows/verify-blog-badge.yml +0 -50
  32. package/.husky/pre-commit +0 -6
  33. package/CHANGELOG.md +0 -131
  34. package/Dockerfile +0 -22
  35. package/Use-Cases.md +0 -134
  36. package/__tests__/fixtures/.ragrc.json +0 -8
  37. package/__tests__/fixtures/invalid-ragrc.json +0 -4
  38. package/__tests__/fixtures/sample-eval-dataset.json +0 -15
  39. package/__tests__/fixtures/sample.pdf +0 -1
  40. package/__tests__/fixtures/src/mocks/openai-embedder.js +0 -16
  41. package/__tests__/fixtures/src/mocks/openai-llm.js +0 -9
  42. package/__tests__/fixtures/src/mocks/pdf-loader.js +0 -17
  43. package/__tests__/fixtures/src/mocks/pinecone-retriever.js +0 -26
  44. package/__tests__/fixtures/valid-ragrc.json +0 -16
  45. package/__tests__/integration/cli/config-flow.test.js +0 -98
  46. package/__tests__/integration/config/load-config.test.js +0 -67
  47. package/__tests__/scripts/ensure-roadmap-labels.test.js +0 -64
  48. package/__tests__/unit/config/validate-schema.test.js +0 -121
  49. package/__tests__/unit/core/plugin-registry.test.js +0 -55
  50. package/__tests__/unit/reranker/__snapshots__/reranker.snapshot.test.js.snap +0 -15
  51. package/__tests__/unit/reranker/llm-reranker.test.js +0 -18
  52. package/__tests__/unit/reranker/reranker.enriched.test.js +0 -19
  53. package/__tests__/unit/reranker/reranker.fallback.test.js +0 -15
  54. package/__tests__/unit/reranker/reranker.snapshot.test.js +0 -15
  55. package/__tests__/unit/reranker/reranker.structured-output.test.js +0 -17
  56. package/codecov +0 -0
  57. package/codecov.SHA256SUM +0 -1
  58. package/codecov.SHA256SUM.sig +0 -16
  59. package/docs-site/README.md +0 -41
  60. package/docs-site/blog/2025-04-18-architecture-overview.md +0 -16
  61. package/docs-site/blog/2025-04-18-release-v2.1.1.md +0 -13
  62. package/docs-site/blog/2025-04-19-prompt-engineering-best-practices.md +0 -16
  63. package/docs-site/blog/2025-04-20-image-showcase.md +0 -14
  64. package/docs-site/blog/2025-04-20-release-2.1.1.md +0 -21
  65. package/docs-site/blog/2025-04-21-release-v2.1.3.md +0 -13
  66. package/docs-site/blog/2025-04-21-release-v2.1.4.md +0 -13
  67. package/docs-site/blog/2025-04-22-release-v2.1.5.md +0 -13
  68. package/docs-site/blog/2025-04-24-v2.1.6.md +0 -21
  69. package/docs-site/blog/authors.yml +0 -5
  70. package/docs-site/blog/tags.yml +0 -60
  71. package/docs-site/docs/Architecture.md +0 -76
  72. package/docs-site/docs/CLI.md +0 -69
  73. package/docs-site/docs/Evaluation.md +0 -61
  74. package/docs-site/docs/FAQ.md +0 -50
  75. package/docs-site/docs/Introduction.md +0 -58
  76. package/docs-site/docs/Plugins.md +0 -61
  77. package/docs-site/docs/Tutorials.md +0 -106
  78. package/docs-site/docs/Usage.md +0 -72
  79. package/docs-site/docs/Use-Cases.md +0 -77
  80. package/docs-site/docs/intro.mdx +0 -6
  81. package/docs-site/docusaurus.config.js +0 -88
  82. package/docs-site/package-lock.json +0 -17914
  83. package/docs-site/package.json +0 -47
  84. package/docs-site/sidebars.js +0 -39
  85. package/docs-site/src/components/HomepageFeatures/index.js +0 -56
  86. package/docs-site/src/components/HomepageFeatures/index.module.css +0 -57
  87. package/docs-site/src/components/HomepageHeader/index.js +0 -46
  88. package/docs-site/src/components/HomepageHeader/index.module.css +0 -77
  89. package/docs-site/src/css/custom.css +0 -104
  90. package/docs-site/src/pages/index.js +0 -20
  91. package/docs-site/src/pages/index.module.css +0 -23
  92. package/docs-site/src/pages/markdown-page.md +0 -7
  93. package/docs-site/static/.nojekyll +0 -0
  94. package/docs-site/static/img/authors/ali.jpg +0 -0
  95. package/docs-site/static/img/hero.jpg +0 -0
  96. package/docs-site/static/img/logo.png +0 -0
  97. package/docs-site/static/img/logo.svg +0 -6
  98. package/docs-site/static/img/social-card.jpg +0 -0
  99. package/jest.config.js +0 -24
  100. package/plugin.config.json +0 -14
  101. package/public/index.html +0 -0
  102. package/public/results/sample-eval.json +0 -0
  103. package/public/src/App.jsx +0 -57
  104. package/release.config.js +0 -36
  105. package/scripts/banner-injector.js +0 -102
  106. package/scripts/ci-runner.js +0 -51
  107. package/scripts/close-done-roadmap-issues.js +0 -65
  108. package/scripts/create-roadmap-issues.js +0 -47
  109. package/scripts/ensure-roadmap-labels.js +0 -73
  110. package/scripts/fix-mdx-blog-imports.js +0 -50
  111. package/scripts/fix-mdx-image-imports.js +0 -27
  112. package/scripts/generate-release-note.js +0 -110
  113. package/scripts/get-previous-tag.js +0 -40
  114. package/scripts/label-roadmap-issues.js +0 -125
  115. package/scripts/manual-test-checklist.md +0 -90
  116. package/scripts/repair-fixtures.js +0 -30
  117. package/scripts/setup.js +0 -31
  118. package/scripts/sync-labels.js +0 -71
  119. package/scripts/sync-roadmap-labels.js +0 -21
  120. package/scripts/tag-release.js +0 -64
  121. package/scripts/test-global-setup.js +0 -18
  122. package/scripts/validate-changelog-version.js +0 -48
  123. package/scripts/verify-fixture-mocks.js +0 -45
  124. package/scripts/verify-fixtures.js +0 -36
  125. package/server.js +0 -31
  126. package/src/config/load-config.js +0 -33
  127. package/src/config/load-plugin-config.js +0 -56
  128. package/src/config/validate-plugin-schema.js +0 -28
  129. package/src/config/validate-schema.js +0 -107
  130. package/src/core/create-pipeline.js +0 -96
  131. package/src/core/plugin-contracts.js +0 -21
  132. package/src/core/plugin-registry.js +0 -48
  133. package/src/dag/dag-engine.js +0 -70
  134. package/src/evaluate/evaluator.js +0 -44
  135. package/src/evaluate/scoring.js +0 -74
  136. package/src/loader/csv-loader.js +0 -30
  137. package/src/loader/directory-loader.js +0 -38
  138. package/src/loader/html-loader.js +0 -57
  139. package/src/loader/markdown-loader.js +0 -58
  140. package/src/mocks/openai-embedder.js +0 -30
  141. package/src/mocks/openai-llm.js +0 -18
  142. package/src/mocks/pdf-loader.js +0 -17
  143. package/src/mocks/pinecone-retriever.js +0 -25
  144. package/src/reranker/llm-reranker.js +0 -34
  145. package/src/utils/ci/diagnostic-reporter.js +0 -36
  146. package/src/utils/logger.js +0 -29
  147. package/src/utils/retry.js +0 -42
  148. package/src/utils/validate-plugin-contract.js +0 -36
@@ -1,77 +0,0 @@
1
- ---
2
- id: Use-Cases
3
- title: Real-World Use Cases
4
- sidebar_position: 5
5
- ---
6
-
7
- ## Real-World Applications of RAG Pipeline Utils
8
-
9
- This project goes beyond traditional RAG tools — it’s a **developer-focused modular framework**. Here's how it’s used:
10
-
11
- ---
12
-
13
- ### 1. Customizable LLM Workflows
14
-
15
- **Use Case:** A team wants to test three different retrievers (Pinecone, Weaviate, Redis) and switch LLMs dynamically during eval.
16
-
17
- ```bash
18
- rag-utils ingest sample.pdf --retriever pinecone --llm openai
19
- ```
20
-
21
- ---
22
-
23
- ### 2. Plugin-Based Evaluation Benchmarks
24
-
25
- **Use Case:** You want to run BLEU/ROUGE scoring across prompt templates or documents using CLI:
26
-
27
- ```bash
28
- rag-utils evaluate --dataset tests/eval.json --llm anthropic
29
- ```
30
-
31
- ---
32
-
33
- ### 3. Internal LLM System for SaaS
34
-
35
- **Use Case:** Embed RAG processing into a backend:
36
-
37
- ```js
38
- import { PluginRegistry, runPipeline } from 'rag-pipeline-utils';
39
-
40
- const registry = new PluginRegistry();
41
- registry.register('embedder', 'openai', new OpenAIEmbedder());
42
-
43
- const output = await runPipeline({
44
- loader: 'pdf',
45
- retriever: 'pinecone',
46
- llm: 'openai',
47
- query: 'How does this work?'
48
- });
49
- ```
50
-
51
- ---
52
-
53
- ### 4. GitHub + NPM Automation for ML Pipelines
54
-
55
- **Use Case:** You want a release blog post + versioned package published automatically:
56
-
57
- - Commit code
58
- - Push to `main`
59
- - GitHub Action triggers:
60
- - Semantic release
61
- - CHANGELOG update
62
- - Blog post generation
63
- - NPM publish
64
-
65
- ---
66
-
67
- ### Benefits
68
-
69
- - **Pluggable** components via clean interfaces
70
- - **CLI + programmatic** access for flexible DX
71
- - **CI-validated** plugin contract enforcement
72
- - **Docs-first** developer onboarding
73
- - **Production-ready** for real ML teams
74
-
75
- ---
76
-
77
- > Want to contribute your use case? PRs welcome on [GitHub](https://github.com/DevilsDev/rag-pipeline-utils).
@@ -1,6 +0,0 @@
1
- ---
2
- id: intro
3
- title: Introduction
4
- ---
5
-
6
- Welcome to RAG Pipeline Utils! This is the starting point for your documentation.
@@ -1,88 +0,0 @@
1
- /**
2
- * Version: 0.4.0
3
- * Path: /docs-site/docusaurus.config.js
4
- * Description: Tailwind integration, sticky navbar, and dark/light toggle support
5
- * Author: Ali Kahwaji
6
- */
7
-
8
- // @ts-check
9
-
10
- export default {
11
- title: 'RAG Pipeline Utils',
12
- tagline: 'Composable pipelines for LLMs',
13
- url: 'https://devilsdev.github.io',
14
- baseUrl: '/rag-pipeline-utils/',
15
- deploymentBranch: 'gh-pages',
16
- onBrokenLinks: 'warn',
17
- onBrokenMarkdownLinks: 'warn',
18
-
19
- favicon: 'img/favicon.ico',
20
-
21
- themeConfig: {
22
- metadata: [
23
- { name: 'keywords', content: 'rag, llm, langchain, pipeline, openai, nodejs' },
24
- { name: 'author', content: 'Ali Kahwaji' },
25
- { name: 'theme-color', content: '#3EF4B6' }
26
- ],
27
- // ✅ OpenGraph / Twitter image
28
- image: 'img/social-card.jpg',
29
-
30
- // ✅ Color mode toggle support
31
- colorMode: {
32
- defaultMode: 'light',
33
- disableSwitch: false,
34
- respectPrefersColorScheme: true
35
- },
36
-
37
- // ✅ Sticky header
38
- navbar: {
39
- title: 'RAG Utils',
40
- hideOnScroll: false,
41
- logo: {
42
- alt: '',
43
- src: '/img/logo.svg'
44
- },
45
- items: [
46
- { to: '/docs/Introduction', label: 'Docs', position: 'left' },
47
- { to: '/blog', label: 'Blog', position: 'left' },
48
- {
49
- href: 'https://github.com/DevilsDev/rag-pipeline-utils',
50
- label: 'GitHub',
51
- position: 'right'
52
- }
53
- ]
54
- },
55
-
56
- footer: {
57
- style: 'dark',
58
- copyright: `© ${new Date().getFullYear()} Ali Kahwaji`
59
- }
60
- },
61
-
62
- organizationName: 'DevilsDev',
63
- projectName: 'rag-pipeline-utils',
64
-
65
- presets: [
66
- [
67
- 'classic',
68
- {
69
- docs: {
70
- sidebarPath: require.resolve('./sidebars.js'),
71
- editUrl:
72
- 'https://github.com/DevilsDev/rag-pipeline-utils/edit/main/docs-site/'
73
- },
74
- blog: {
75
- showReadingTime: true,
76
- blogSidebarTitle: 'All Posts',
77
- blogSidebarCount: 'ALL',
78
- onUntruncatedBlogPosts: 'ignore',
79
- editUrl:
80
- 'https://github.com/DevilsDev/rag-pipeline-utils/edit/main/docs-site/blog/'
81
- },
82
- theme: {
83
- customCss: require.resolve('./src/css/custom.css')
84
- }
85
- }
86
- ]
87
- ]
88
- };