@devilsdev/rag-pipeline-utils 2.1.7 → 2.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/LICENSE +674 -674
  2. package/README.md +395 -186
  3. package/bin/cli.js +14 -62
  4. package/contracts/embedder-contract.json +52 -0
  5. package/contracts/llm-contract.json +36 -0
  6. package/contracts/loader-contract.json +43 -0
  7. package/contracts/retriever-contract.json +68 -0
  8. package/dist/index.cjs +67 -0
  9. package/dist/index.mjs +51 -0
  10. package/package.json +108 -23
  11. package/.codecov.yml +0 -21
  12. package/.dockerignore +0 -14
  13. package/.eslintignore +0 -5
  14. package/.eslintrc.json +0 -28
  15. package/.github/PROJECT_ROADMAP.md +0 -102
  16. package/.github/PULL_REQUEST_TEMPLATE.md +0 -8
  17. package/.github/release-template.md +0 -20
  18. package/.github/workflows/auto-release-note.yml +0 -73
  19. package/.github/workflows/ci.yml +0 -55
  20. package/.github/workflows/docs-ci.yml +0 -46
  21. package/.github/workflows/docs-deploy.yml +0 -64
  22. package/.github/workflows/enforce-release-review.yml +0 -50
  23. package/.github/workflows/generate-blog-post.yml +0 -44
  24. package/.github/workflows/post-release-generate-blog.yml +0 -71
  25. package/.github/workflows/publish-release-blog.yml +0 -47
  26. package/.github/workflows/release-sync.yml +0 -50
  27. package/.github/workflows/roadmap-label-sync.yml +0 -35
  28. package/.github/workflows/roadmap-labels.yml +0 -67
  29. package/.github/workflows/roadmap-sync.yml +0 -44
  30. package/.github/workflows/sync-roadmap.yml +0 -62
  31. package/.github/workflows/verify-blog-badge.yml +0 -50
  32. package/.husky/pre-commit +0 -6
  33. package/CHANGELOG.md +0 -131
  34. package/Dockerfile +0 -22
  35. package/Use-Cases.md +0 -134
  36. package/__tests__/fixtures/.ragrc.json +0 -8
  37. package/__tests__/fixtures/invalid-ragrc.json +0 -4
  38. package/__tests__/fixtures/sample-eval-dataset.json +0 -15
  39. package/__tests__/fixtures/sample.pdf +0 -1
  40. package/__tests__/fixtures/src/mocks/openai-embedder.js +0 -16
  41. package/__tests__/fixtures/src/mocks/openai-llm.js +0 -9
  42. package/__tests__/fixtures/src/mocks/pdf-loader.js +0 -17
  43. package/__tests__/fixtures/src/mocks/pinecone-retriever.js +0 -26
  44. package/__tests__/fixtures/valid-ragrc.json +0 -16
  45. package/__tests__/integration/cli/config-flow.test.js +0 -98
  46. package/__tests__/integration/config/load-config.test.js +0 -67
  47. package/__tests__/scripts/ensure-roadmap-labels.test.js +0 -64
  48. package/__tests__/unit/config/validate-schema.test.js +0 -121
  49. package/__tests__/unit/core/plugin-registry.test.js +0 -55
  50. package/__tests__/unit/reranker/__snapshots__/reranker.snapshot.test.js.snap +0 -15
  51. package/__tests__/unit/reranker/llm-reranker.test.js +0 -18
  52. package/__tests__/unit/reranker/reranker.enriched.test.js +0 -19
  53. package/__tests__/unit/reranker/reranker.fallback.test.js +0 -15
  54. package/__tests__/unit/reranker/reranker.snapshot.test.js +0 -15
  55. package/__tests__/unit/reranker/reranker.structured-output.test.js +0 -17
  56. package/codecov +0 -0
  57. package/codecov.SHA256SUM +0 -1
  58. package/codecov.SHA256SUM.sig +0 -16
  59. package/docs-site/README.md +0 -41
  60. package/docs-site/blog/2025-04-18-architecture-overview.md +0 -16
  61. package/docs-site/blog/2025-04-18-release-v2.1.1.md +0 -13
  62. package/docs-site/blog/2025-04-19-prompt-engineering-best-practices.md +0 -16
  63. package/docs-site/blog/2025-04-20-image-showcase.md +0 -14
  64. package/docs-site/blog/2025-04-20-release-2.1.1.md +0 -21
  65. package/docs-site/blog/2025-04-21-release-v2.1.3.md +0 -13
  66. package/docs-site/blog/2025-04-21-release-v2.1.4.md +0 -13
  67. package/docs-site/blog/2025-04-22-release-v2.1.5.md +0 -13
  68. package/docs-site/blog/2025-04-24-v2.1.6.md +0 -21
  69. package/docs-site/blog/authors.yml +0 -5
  70. package/docs-site/blog/tags.yml +0 -60
  71. package/docs-site/docs/Architecture.md +0 -76
  72. package/docs-site/docs/CLI.md +0 -69
  73. package/docs-site/docs/Evaluation.md +0 -61
  74. package/docs-site/docs/FAQ.md +0 -50
  75. package/docs-site/docs/Introduction.md +0 -58
  76. package/docs-site/docs/Plugins.md +0 -61
  77. package/docs-site/docs/Tutorials.md +0 -106
  78. package/docs-site/docs/Usage.md +0 -72
  79. package/docs-site/docs/Use-Cases.md +0 -77
  80. package/docs-site/docs/intro.mdx +0 -6
  81. package/docs-site/docusaurus.config.js +0 -88
  82. package/docs-site/package-lock.json +0 -17914
  83. package/docs-site/package.json +0 -47
  84. package/docs-site/sidebars.js +0 -39
  85. package/docs-site/src/components/HomepageFeatures/index.js +0 -56
  86. package/docs-site/src/components/HomepageFeatures/index.module.css +0 -57
  87. package/docs-site/src/components/HomepageHeader/index.js +0 -46
  88. package/docs-site/src/components/HomepageHeader/index.module.css +0 -77
  89. package/docs-site/src/css/custom.css +0 -104
  90. package/docs-site/src/pages/index.js +0 -20
  91. package/docs-site/src/pages/index.module.css +0 -23
  92. package/docs-site/src/pages/markdown-page.md +0 -7
  93. package/docs-site/static/.nojekyll +0 -0
  94. package/docs-site/static/img/authors/ali.jpg +0 -0
  95. package/docs-site/static/img/hero.jpg +0 -0
  96. package/docs-site/static/img/logo.png +0 -0
  97. package/docs-site/static/img/logo.svg +0 -6
  98. package/docs-site/static/img/social-card.jpg +0 -0
  99. package/jest.config.js +0 -24
  100. package/plugin.config.json +0 -14
  101. package/public/index.html +0 -0
  102. package/public/results/sample-eval.json +0 -0
  103. package/public/src/App.jsx +0 -57
  104. package/release.config.js +0 -36
  105. package/scripts/banner-injector.js +0 -102
  106. package/scripts/ci-runner.js +0 -51
  107. package/scripts/close-done-roadmap-issues.js +0 -65
  108. package/scripts/create-roadmap-issues.js +0 -47
  109. package/scripts/ensure-roadmap-labels.js +0 -73
  110. package/scripts/fix-mdx-blog-imports.js +0 -50
  111. package/scripts/fix-mdx-image-imports.js +0 -27
  112. package/scripts/generate-release-note.js +0 -110
  113. package/scripts/get-previous-tag.js +0 -40
  114. package/scripts/label-roadmap-issues.js +0 -125
  115. package/scripts/manual-test-checklist.md +0 -90
  116. package/scripts/repair-fixtures.js +0 -30
  117. package/scripts/setup.js +0 -31
  118. package/scripts/sync-labels.js +0 -71
  119. package/scripts/sync-roadmap-labels.js +0 -21
  120. package/scripts/tag-release.js +0 -64
  121. package/scripts/test-global-setup.js +0 -18
  122. package/scripts/validate-changelog-version.js +0 -48
  123. package/scripts/verify-fixture-mocks.js +0 -45
  124. package/scripts/verify-fixtures.js +0 -36
  125. package/server.js +0 -31
  126. package/src/config/load-config.js +0 -33
  127. package/src/config/load-plugin-config.js +0 -56
  128. package/src/config/validate-plugin-schema.js +0 -28
  129. package/src/config/validate-schema.js +0 -107
  130. package/src/core/create-pipeline.js +0 -96
  131. package/src/core/plugin-contracts.js +0 -21
  132. package/src/core/plugin-registry.js +0 -48
  133. package/src/dag/dag-engine.js +0 -70
  134. package/src/evaluate/evaluator.js +0 -44
  135. package/src/evaluate/scoring.js +0 -74
  136. package/src/loader/csv-loader.js +0 -30
  137. package/src/loader/directory-loader.js +0 -38
  138. package/src/loader/html-loader.js +0 -57
  139. package/src/loader/markdown-loader.js +0 -58
  140. package/src/mocks/openai-embedder.js +0 -30
  141. package/src/mocks/openai-llm.js +0 -18
  142. package/src/mocks/pdf-loader.js +0 -17
  143. package/src/mocks/pinecone-retriever.js +0 -25
  144. package/src/reranker/llm-reranker.js +0 -34
  145. package/src/utils/ci/diagnostic-reporter.js +0 -36
  146. package/src/utils/logger.js +0 -29
  147. package/src/utils/retry.js +0 -42
  148. package/src/utils/validate-plugin-contract.js +0 -36
@@ -1,67 +0,0 @@
1
- # Version: 1.0.0
2
- # Description: Ensures roadmap labels (priority, group) exist in the repo
3
- # Author: Ali Kahwaji
4
-
5
- name: 🔖 Sync Roadmap Labels
6
-
7
- on:
8
- workflow_dispatch:
9
- push:
10
- paths:
11
- - '.github/workflows/roadmap-labels.yml'
12
-
13
- jobs:
14
- sync-labels:
15
- runs-on: ubuntu-latest
16
-
17
- steps:
18
- - name: ⬇️ Checkout repository
19
- uses: actions/checkout@v4
20
-
21
- - name: 🟢 Setup Node.js
22
- uses: actions/setup-node@v4
23
- with:
24
- node-version: 20
25
-
26
- - name: 📦 Install octokit
27
- run: npm install octokit
28
-
29
- - name: 🔖 Run label sync
30
- env:
31
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
32
- GITHUB_REPO: ${{ github.repository }}
33
- run: |
34
- node <<EOF
35
- const { Octokit } = require("octokit");
36
-
37
- const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
38
- const [owner, repo] = process.env.GITHUB_REPO.split("/");
39
-
40
- const labels = [
41
- { name: "priority: high", color: "e11d48", description: "High priority roadmap item" },
42
- { name: "priority: medium", color: "f59e0b", description: "Medium priority roadmap item" },
43
- { name: "priority: low", color: "10b981", description: "Low priority roadmap item" },
44
- { name: "group: docs", color: "6366f1", description: "Documentation features" },
45
- { name: "group: devx", color: "06b6d4", description: "Developer experience improvements" },
46
- { name: "group: community", color: "ec4899", description: "Community tools & engagement" },
47
- { name: "group: blog", color: "f97316", description: "Blog & SEO enhancements" },
48
- { name: "group: infra", color: "64748b", description: "Infrastructure & deployment features" }
49
- ];
50
-
51
- (async () => {
52
- const existing = await octokit.rest.issues.listLabelsForRepo({ owner, repo });
53
- const existingNames = new Set(existing.data.map(l => l.name));
54
-
55
- for (const label of labels) {
56
- if (!existingNames.has(label.name)) {
57
- console.log(`➕ Creating label: ${label.name}`);
58
- await octokit.rest.issues.createLabel({ owner, repo, ...label });
59
- } else {
60
- console.log(`✅ Label already exists: ${label.name}`);
61
- }
62
- }
63
- })().catch(err => {
64
- console.error("Label sync failed:", err);
65
- process.exit(1);
66
- });
67
- EOF
@@ -1,44 +0,0 @@
1
- name: 🚀 Roadmap Sync (Labels + Issues)
2
-
3
- on:
4
- push:
5
- branches: [main]
6
- paths:
7
- - '.github/roadmap-labels.yml'
8
- - '.github/PROJECT_ROADMAP.md'
9
- - 'scripts/create-roadmap-issues.js'
10
- - 'scripts/ensure-roadmap-labels.js'
11
-
12
- workflow_dispatch:
13
-
14
- permissions:
15
- issues: write
16
- contents: read
17
-
18
- jobs:
19
- roadmap-sync:
20
- name: 🔁 Sync Labels + Issues from Roadmap
21
- runs-on: ubuntu-latest
22
-
23
- steps:
24
- - name: 📥 Checkout repository
25
- uses: actions/checkout@v4
26
-
27
- - name: 🛠 Setup Node.js
28
- uses: actions/setup-node@v4
29
- with:
30
- node-version: 20
31
- cache: 'npm'
32
-
33
- - name: 📦 Install dependencies
34
- run: npm ci
35
-
36
- - name: 🔖 Sync GitHub labels from roadmap-labels.yml
37
- run: node scripts/ensure-roadmap-labels.js
38
- env:
39
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
40
-
41
- - name: 📝 Create or update roadmap issues
42
- run: node scripts/create-roadmap-issues.js
43
- env:
44
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -1,62 +0,0 @@
1
- # .github/workflows/sync-roadmap.yml
2
- # Version: 2.1.0
3
- # Description: Unified workflow to sync roadmap issues, labels, and states
4
- # Author: Ali Kahwaji
5
-
6
- name: 🚀 Sync Roadmap to GitHub Issues
7
-
8
- on:
9
- workflow_dispatch:
10
- push:
11
- branches: [main]
12
- paths:
13
- - '.github/PROJECT_ROADMAP.md'
14
- - '.github/workflows/sync-roadmap.yml'
15
- - 'scripts/**.js'
16
-
17
- permissions:
18
- contents: write
19
- issues: write
20
-
21
- jobs:
22
- sync:
23
- name: 📌 Roadmap Sync
24
- runs-on: ubuntu-latest
25
-
26
- steps:
27
- - name: ⬇️ Checkout
28
- uses: actions/checkout@v4
29
-
30
- - name: 🛠 Setup Node.js
31
- uses: actions/setup-node@v4
32
- with:
33
- node-version: 20
34
-
35
- - name: 📦 Install roadmap dependencies
36
- run: npm install
37
- env:
38
- NODE_ENV: development
39
-
40
- - name: 🔖 Ensure roadmap labels exist
41
- run: node scripts/ensure-roadmap-labels.js
42
- env:
43
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44
- GITHUB_REPO: ${{ github.repository }}
45
-
46
- - name: 🗂 Sync roadmap issues
47
- run: node scripts/create-roadmap-issues.js
48
- env:
49
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
50
- GITHUB_REPO: ${{ github.repository }}
51
-
52
- - name: 🏷 Apply labels to roadmap issues
53
- run: node scripts/label-roadmap-issues.js
54
- env:
55
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
56
- GITHUB_REPO: ${{ github.repository }}
57
-
58
- - name: ✅ Close completed issues
59
- run: node scripts/close-done-roadmap-issues.js
60
- env:
61
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
62
- GITHUB_REPO: ${{ github.repository }}
@@ -1,50 +0,0 @@
1
- # .github/workflows/verify-blog-badge.yml
2
- # Version: 1.3.0
3
- # Description: Validates presence of blog generation badge + GitHub Action output
4
- # Author: Ali Kahwaji
5
-
6
- name: 🖍 Blog Badge Verification
7
-
8
- on:
9
- workflow_dispatch:
10
- workflow_run:
11
- workflows: ["Auto Generate Blog + Changelog"]
12
- types:
13
- - completed
14
-
15
- jobs:
16
- verify:
17
- name: 🌟 Verify Blog Badge and Output
18
- runs-on: ubuntu-latest
19
- if: ${{ github.event.workflow_run.conclusion == 'success' }}
20
-
21
- steps:
22
- - name: ⬇️ Checkout repository
23
- uses: actions/checkout@v4
24
-
25
- - name: 🔍 Validate README badge reference
26
- run: |
27
- echo "Checking for badge reference in README.md..."
28
- grep -q "post-release-generate-blog.yml/badge.svg" README.md || {
29
- echo "❌ Missing badge for post-release-generate-blog.yml in README.md"
30
- exit 1
31
- }
32
-
33
- - name: 🌐 Validate badge URL status
34
- run: |
35
- BADGE_URL="https://github.com/DevilsDev/rag-pipeline-utils/actions/workflows/post-release-generate-blog.yml/badge.svg?branch=main"
36
- STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BADGE_URL")
37
-
38
- echo "Badge response: $STATUS"
39
- if [ "$STATUS" != "200" ]; then
40
- echo "❌ Blog badge not reachable (HTTP $STATUS)"
41
- exit 1
42
- fi
43
-
44
- - name: 🧾 Log assumed blog generation output (manual mapping)
45
- run: |
46
- echo "⚠️ Outputs from 'workflow_run' do not include job outputs."
47
- echo "Assuming blog_generated=true based on successful workflow completion."
48
-
49
- - name: ✅ Blog badge verified
50
- run: echo "✅ Blog badge verified and present in README."
package/.husky/pre-commit DELETED
@@ -1,6 +0,0 @@
1
- #!/usr/bin/env sh
2
- . "$(dirname -- "$0")/_/husky.sh"
3
-
4
- npm run validate-fixtures && npm run lint
5
- npx lint-staged
6
- node scripts/validate-changelog-version.js
package/CHANGELOG.md DELETED
@@ -1,131 +0,0 @@
1
- # Changelog
2
-
3
- ## [2.1.7](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.1.6...v2.1.7) (2025-04-25)
4
-
5
-
6
- ### Bug Fixes
7
-
8
- * **ci:** patch blog release workflow with DRY_RUN and safer outputs ([18f1b7d](https://github.com/DevilsDev/rag-pipeline-utils/commit/18f1b7dd4019022d76de9097db1ff8cfd858d83e))
9
-
10
- <<<<<<< HEAD
11
- ## 2.1.5 - 2025-04-22
12
- ### Features
13
- - Synced roadmap labels + issues via CI
14
- - Added auto-close for Done roadmap entries
15
-
16
- ### Fixes
17
- - ES module compatibility fixes for roadmap scripts
18
- - Improved test isolation and mocking for Octokit
19
-
20
- ### DevOps
21
- - CI chain includes roadmap validation and label sync
22
- - CLI tools support changelog generation + blog linking
23
-
24
-
25
- =======
26
- ## [2.1.5](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.1.4...v2.1.5) (2025-04-22)
27
-
28
-
29
- ### Bug Fixes
30
-
31
- * **scripts:** prevent null status crash in roadmap issue closer ([89d5a0b](https://github.com/DevilsDev/rag-pipeline-utils/commit/89d5a0bc4cac554975974558b97974297816f169))
32
-
33
- >>>>>>> 6475a97d362f7cd9779e1268f1c816764a7fd62c
34
- ## [2.1.4](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.1.3...v2.1.4) (2025-04-21)
35
-
36
-
37
- ### Bug Fixes
38
-
39
- * update CI and deployment workflows for docs build and GitHub Pages deployment ([11739ba](https://github.com/DevilsDev/rag-pipeline-utils/commit/11739ba222669e359232807e461bc84d68d913cb))
40
- * update CI and deployment workflows for docs build and GitHub Pages deployment ([8de276f](https://github.com/DevilsDev/rag-pipeline-utils/commit/8de276f8cb6c91a291322783dc944b0f4799974f))
41
-
42
- ## [2.1.3](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.1.2...v2.1.3) (2025-04-21)
43
-
44
-
45
- ### Bug Fixes
46
-
47
- * Refactor CI workflows for better separation of concerns ([ef2954b](https://github.com/DevilsDev/rag-pipeline-utils/commit/ef2954b9161a73367cafa04d29f2e56164e4b1c0))
48
-
49
- ## [2.1.2](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.1.1...v2.1.2) (2025-04-21)
50
-
51
-
52
- ### Bug Fixes
53
-
54
- * resolve build/test errors, update jest configuration, and ensure compatibility with ECMAScript modules ([e6259f9](https://github.com/DevilsDev/rag-pipeline-utils/commit/e6259f924f38da66f9fc9cb7c01a8d6665bf5cbd))
55
-
56
- ## [2.1.1](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.1.0...v2.1.1) (2025-04-18)
57
-
58
-
59
- ### Bug Fixes
60
-
61
- * **blog:** convert tags.yml to valid object format for Docusaurus v3 ([6a7a9bf](https://github.com/DevilsDev/rag-pipeline-utils/commit/6a7a9bfa3501d30529b21e36fa51ab47d151f7a3))
62
- * **blog:** convert tags.yml to valid object format for Docusaurus v3 ([fa2fff9](https://github.com/DevilsDev/rag-pipeline-utils/commit/fa2fff969dc242e66d92a80b4aa2ff8e07f9fd0f))
63
-
64
- ## [2.1.0] - 2025-04-18 (2025-04-18)
65
-
66
-
67
- ### Features
68
-
69
- * **docs-site:** add Storybook integration with BlogCard + LQIP image automation ([80965b8](https://github.com/DevilsDev/rag-pipeline-utils/commit/80965b8ef1d26372852ee0f39b56a73f337e0cea))
70
-
71
- ## [2.0.1](https://github.com/DevilsDev/rag-pipeline-utils/compare/v2.0.0...v2.0.1) (2025-04-17)
72
-
73
-
74
- ### Bug Fixes
75
-
76
- * add @docusaurus/preset-classic to enable CI builds ([ed5e771](https://github.com/DevilsDev/rag-pipeline-utils/commit/ed5e771f919face4693a6a7abfed4cf0fe1325ca))
77
-
78
- ## [2.0.0] - 2025-04-17 (2025-04-17)
79
-
80
-
81
- ### Bug Fixes
82
-
83
- * **ci:** correct .ragrc.json plugin paths and CLI test cwd to resolve plugin loading in CI ([911c23b](https://github.com/DevilsDev/rag-pipeline-utils/commit/911c23b1d3ddb9e2d938b37f4422a5d81995e97f))
84
- * **ci:** resolve plugin path error in .ragrc.json for GitHub Actions compatibility ([affc739](https://github.com/DevilsDev/rag-pipeline-utils/commit/affc739d5c4ff2070b1a1ca0ccf3f9137d6cee5a))
85
- * **ci:** resolve plugin path failures in .ragrc.json during CLI integration tests ([04b977b](https://github.com/DevilsDev/rag-pipeline-utils/commit/04b977b3b745fa3d3ab47c84b2aeee3285896068))
86
- * **cli:** correct path to load-plugin-config.js after moving to /src/config ([f743498](https://github.com/DevilsDev/rag-pipeline-utils/commit/f743498ec32e03151eda9ba7fe5886bda8da792f))
87
- * **config:** use default export when importing plugin registry ([ef8dce1](https://github.com/DevilsDev/rag-pipeline-utils/commit/ef8dce12bfab62bc9faec764198b98c0c356472e))
88
- * **test:** ensure .ragrc.json copied to project root for CLI fallback in CI ([31bd2b8](https://github.com/DevilsDev/rag-pipeline-utils/commit/31bd2b844279fb0404056d9929e5f28f9127eb12))
89
- * **tests:** define __dirname manually for ESM compatibility in CLI integration test ([18b8dad](https://github.com/DevilsDev/rag-pipeline-utils/commit/18b8dadabb852243dda2b78de048c71106a759fc))
90
-
91
-
92
- ### Features
93
-
94
- * **ci:** integrate full fixture validation, schema contracts, and CLI test flow ([2b57fe9](https://github.com/DevilsDev/rag-pipeline-utils/commit/2b57fe90e64a806815c1f66f5e2a745780270ce6))
95
- * **cli:** dynamic plugin registration with config fallback and middleware injection ([cc24b7e](https://github.com/DevilsDev/rag-pipeline-utils/commit/cc24b7e74d0a2630eb1c6d2def69b370329dcc2b))
96
- * **plugins:** enable dynamic plugin registration from JSON config ([d81d6d7](https://github.com/DevilsDev/rag-pipeline-utils/commit/d81d6d7d5f2d21b5a46b379b530c162ba9c30398))
97
- * **testing:** integrate full fixture and validation setup for CLI + reranker ([ab96037](https://github.com/DevilsDev/rag-pipeline-utils/commit/ab9603765594e7718e2688b80797e1f5e8afe86f))
98
-
99
-
100
- ### BREAKING CHANGES
101
-
102
- * **cli:** now requires named options for middleware injection; existing usage must be updated.
103
-
104
- ## [1.0.2] - 2025-04-15
105
- ### Fixed
106
- - Corrected plugin path resolution in `.ragrc.json` fixture for CI compatibility.
107
- - Resolved `ERR_MODULE_NOT_FOUND` during CLI integration tests in GitHub Actions.
108
- - Ensured consistent module resolution across environments (Windows, Linux, CI).
109
-
110
-
111
- ## [1.0.1](https://github.com/DevilsDev/rag-pipeline-utils/compare/v1.0.0...v1.0.1) (2025-04-14)
112
-
113
-
114
- ### Bug Fixes
115
-
116
- * **ci:** initialize release pipeline and scaffold changelog ([179b486](https://github.com/DevilsDev/rag-pipeline-utils/commit/179b486effb0751b8735a9f849825eb63930ad01))
117
- * **tests:** add missing sample PDF fixture and pre-test setup script ([de8e2b2](https://github.com/DevilsDev/rag-pipeline-utils/commit/de8e2b2c19034b521fc5ae1cdebbe3884a2fd15f))
118
-
119
- All notable changes to this project will be documented in this file.
120
-
121
- The format is based on [Semantic Versioning](https://semver.org/), and this project uses [semantic-release](https://github.com/semantic-release/semantic-release) for automated versioning and changelog generation.
122
-
123
- ## v2.1.6
124
-
125
- - chore: remove legacy changelog workflow and finalize blog badge
126
- - ci(workflows): add blog badge verification workflow
127
- - ci(workflows): add blog badge verification workflow
128
- - ci(workflows): add blog badge verification workflow
129
- - docs(blog): add release post for v2.1.5
130
-
131
- ---
package/Dockerfile DELETED
@@ -1,22 +0,0 @@
1
- # Version: 0.1.0
2
- # Path: /Dockerfile
3
- # Description: Minimal Docker image for rag-pipeline CLI
4
- # Author: Ali Kahwaji
5
-
6
- FROM node:18-slim
7
-
8
- # Create app directory
9
- WORKDIR /app
10
-
11
- # Copy package files and install deps
12
- COPY package*.json ./
13
- RUN npm ci --omit=dev
14
-
15
- # Copy application source
16
- COPY . .
17
-
18
- # Set default entrypoint
19
- ENTRYPOINT ["node", "./bin/cli.js"]
20
-
21
- # Default help
22
- CMD ["--help"]
package/Use-Cases.md DELETED
@@ -1,134 +0,0 @@
1
- # RAG Pipeline Utils — Developer-Focused Use Cases
2
-
3
- > Modular, composable, and CI-verified RAG orchestration for modern AI applications.
4
-
5
- ---
6
-
7
- ## Use Case 1: Pluggable Local RAG Stack for Prototyping
8
-
9
- **Problem:** Traditional RAG tools force tight vendor dependencies.
10
-
11
- **Solution:** Use mockable local plugin implementations to simulate loaders, embedders, retrievers, and LLMs.
12
-
13
- ```bash
14
- npm install @devilsdev/rag-pipeline-utils
15
- ```
16
-
17
- ```js
18
- // cli.js or app.js
19
- import { PluginRegistry } from './src/core/plugin-registry.js';
20
- import { loadRagConfig } from './src/config/load-config.js';
21
-
22
- const config = loadRagConfig('.ragrc.json');
23
- const loader = PluginRegistry.getPlugin('loader', 'pdf');
24
- const docs = loader.load('./test.pdf');
25
- ```
26
-
27
- **Benefits**
28
- - Local plugin mocks with expected method contracts
29
- - Reusable plugin templates (`pdf-loader.js`, `openai-embedder.js`)
30
- - Dev-first DX: debug, snapshot, iterate without cloud costs
31
-
32
- ---
33
-
34
- ## Use Case 2: Declarative CLI RAG Pipelines (CI-Ready)
35
-
36
- **Problem:** Hardcoded pipeline steps can't be validated or changed by non-engineers.
37
-
38
- **Solution:** Define `.ragrc.json` once and reuse across environments.
39
-
40
- ```json
41
- {
42
- "loader": { "pdf": "./src/mocks/pdf-loader.js" },
43
- "embedder": { "openai": "./src/mocks/openai-embedder.js" },
44
- "retriever": { "pinecone": "./src/mocks/pinecone-retriever.js" },
45
- "llm": { "openai": "./src/mocks/openai-llm.js" },
46
- "namespace": "docs-poc",
47
- "pipeline": ["loader", "embedder", "retriever"]
48
- }
49
- ```
50
-
51
- ```bash
52
- node ./bin/cli.js ingest ./docs/whitepaper.pdf
53
- node ./bin/cli.js query "What’s the goal of this architecture?"
54
- ```
55
-
56
- **Benefits**
57
- - Teams can standardize pipelines in source control
58
- - Tests fail fast if structure or mocks drift
59
- - Non-devs can operate pipelines via CLI
60
-
61
- ---
62
-
63
- ## Use Case 3: Mock-First TDD for Plugin Development
64
-
65
- **Problem:** LLM, vector, and loader logic are often developed without tests.
66
-
67
- **Solution:** Define contract-compliant mocks and CI-validate their structure before connecting to APIs.
68
-
69
- ```js
70
- export default class CustomEmbedder {
71
- embed(docs) { return docs.map(...); }
72
- embedQuery(q) { return [...]; }
73
- }
74
- ```
75
-
76
- Run full contract verification:
77
-
78
- ```bash
79
- npm run ci
80
- ```
81
-
82
- **Benefits**
83
- - Every plugin is CI-verified for method presence
84
- - Shared contracts enforce required interfaces
85
- - Onboarding new devs? Just follow `pluginContracts.js`
86
-
87
- ---
88
-
89
- ## Use Case 4: Vendor-Neutral AI Data Pipelines
90
-
91
- **Problem:** Every RAG stack is locked to OpenAI + Pinecone + Langchain.
92
-
93
- **Solution:** Use this project to mix, match, or mock vector/LLM pairs:
94
-
95
- ```js
96
- "retriever": { "localstore": "./src/plugins/local-retriever.js" },
97
- "llm": { "cohere": "./src/plugins/cohere-llm.js" }
98
- ```
99
-
100
- **Benefits**
101
- - Run multiple RAG pipelines in parallel (A/B testing, evals)
102
- - Lower infra cost with offline/local plugin support
103
- - Integrate with tools outside Langchain ecosystem
104
-
105
- ---
106
-
107
- ## 🛠 Use Case 5: Plugin Engineering with Guardrails
108
-
109
- **Problem:** Complex plugin systems break silently.
110
-
111
- **Solution:** This framework has:
112
- - Plugin method contract enforcement
113
- - CLI test runner (`scripts/ci-runner.js`)
114
- - Auto repair tool (`scripts/repair-fixtures.js`)
115
-
116
- **DX Boost**
117
- - One-line plugin repairs
118
- - Validated schema structure
119
- - Aligned to real-world developer workflows
120
-
121
- ---
122
-
123
- ## Cross References
124
-
125
- - [README](./README.md) — Project overview, architecture, CLI and configuration details
126
- - [pluginContracts.js](./src/core/plugin-contracts.js) — Interface contracts for loader, embedder, retriever, LLM
127
- - [diagnostic-reporter.js](./src/utils/ci/diagnostic-reporter.js) — CI error helper
128
- - [repair-fixtures.js](./scripts/repair-fixtures.js) — Auto-regeneration of mocks
129
-
130
- ---
131
-
132
- ## License
133
-
134
- Licensed under the **GPL-3.0 License** — see [LICENSE](./LICENSE) for full terms.
@@ -1,8 +0,0 @@
1
- {
2
- "loader": { "pdf": "./src/mocks/pdf-loader.js" },
3
- "embedder": { "openai": "./src/mocks/openai-embedder.js" },
4
- "retriever": { "pinecone": "./src/mocks/pinecone-retriever.js" },
5
- "llm": { "openai": "./src/mocks/openai-llm.js" },
6
- "namespace": "cli-config-test",
7
- "pipeline": ["loader", "embedder", "retriever"]
8
- }
@@ -1,4 +0,0 @@
1
- {
2
- "loader": "./src/mocks/pdf-loader.js",
3
- "namespace": 12345
4
- }
@@ -1,15 +0,0 @@
1
- [
2
- {
3
- "prompt": "What is a vector database?",
4
- "expected": "A vector database is a system optimized for storing and retrieving data represented as vectors, often used in similarity search."
5
- },
6
- {
7
- "prompt": "Explain what chunking means in RAG.",
8
- "expected": "Chunking is the process of splitting documents into smaller, manageable pieces for embedding and retrieval in a RAG pipeline."
9
- },
10
- {
11
- "prompt": "What is the role of embedding in a RAG pipeline?",
12
- "expected": "Embeddings convert text into numerical vector form so they can be compared in a vector database during retrieval."
13
- }
14
- ]
15
-
@@ -1 +0,0 @@
1
- %PDF-1.4\n%âãÏÓ\n1 0 obj\n<<>>\nendobj\nxref\n0 2\n0000000000 65535 f\n0000000010 00000 n\ntrailer\n<<>>\nstartxref\n9\n%%EOF
@@ -1,16 +0,0 @@
1
- /**
2
- * Mock OpenAI Embedder Plugin
3
- * Implements: embedder.embed(docs), embedder.embedQuery(query)
4
- */
5
- export default class OpenAIEmbedder {
6
- embed(documents) {
7
- return documents.map(doc => ({
8
- id: doc.id,
9
- values: [0.1, 0.2, 0.3],
10
- }));
11
- }
12
-
13
- embedQuery(query) {
14
- return [0.1, 0.2, 0.3];
15
- }
16
- }
@@ -1,9 +0,0 @@
1
- /**
2
- * Mock OpenAI LLM Plugin
3
- * Implements: llm.ask(prompt)
4
- */
5
- export default class OpenAILLM {
6
- ask(prompt) {
7
- return `Answer: Mocked response to "${prompt}"`;
8
- }
9
- }
@@ -1,17 +0,0 @@
1
- /**
2
- * Mock PDF Loader Plugin
3
- * Implements: loader.load(filePath)
4
- */
5
- export default class PdfLoader {
6
- /**
7
- * Loads a file and returns mock document chunks.
8
- * @param {string} filePath
9
- * @returns {Array<{ id: string, content: string }>}
10
- */
11
- load(filePath) {
12
- return [
13
- { id: 'doc1', content: `Loaded content from ${filePath}` },
14
- { id: 'doc2', content: 'Second document content' },
15
- ];
16
- }
17
- }
@@ -1,26 +0,0 @@
1
- /**
2
- * Mock Pinecone Retriever Plugin
3
- * Implements: retriever.store(vectors), retriever.search(queryVector)
4
- */
5
- export default class PineconeRetriever {
6
- constructor() {
7
- this._store = [];
8
- }
9
-
10
- /**
11
- * Stores embedded vectors.
12
- * @param {Array<{ id: string, values: number[] }>} vectors
13
- */
14
- store(vectors) {
15
- this._store = vectors;
16
- }
17
-
18
- /**
19
- * Searches mock memory and returns matching docs.
20
- * @param {number[]} queryVector
21
- * @returns {Array<{ id: string, score: number }>}
22
- */
23
- search(queryVector) {
24
- return this._store.map(v => ({ id: v.id, score: 0.99 }));
25
- }
26
- }
@@ -1,16 +0,0 @@
1
- {
2
- "loader": {
3
- "pdf": "./src/mocks/pdf-loader.js"
4
- },
5
- "embedder": {
6
- "openai": "./src/mocks/openai-embedder.js"
7
- },
8
- "retriever": {
9
- "pinecone": "./src/mocks/pinecone-retriever.js"
10
- },
11
- "llm": {
12
- "openai": "./src/mocks/openai-llm.js"
13
- },
14
- "namespace": "cli-config-test",
15
- "pipeline": ["loader", "embedder", "retriever"]
16
- }