@devilsdev/rag-pipeline-utils 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +14 -0
- package/.github/workflows/ci.yml +42 -0
- package/.github/workflows/release.yml +21 -0
- package/Dockerfile +22 -0
- package/LICENSE +674 -0
- package/README.md +116 -0
- package/__tests__/fixtures/sample-eval-dataset.json +15 -0
- package/__tests__/integration/cli/config-flow.test.js +44 -0
- package/__tests__/integration/config/load-config.test.js +52 -0
- package/__tests__/unit/core/plugin-registry.test.js +55 -0
- package/__tests__/unit/reranker/__snapshots__/reranker.snapshot.test.js.snap +15 -0
- package/__tests__/unit/reranker/llm-reranker.test.js +42 -0
- package/__tests__/unit/reranker/reranker.snapshot.test.js +17 -0
- package/bin/cli.js +168 -0
- package/docs-site/README.md +41 -0
- package/docs-site/blog/2019-05-28-first-blog-post.md +12 -0
- package/docs-site/blog/2019-05-29-long-blog-post.md +44 -0
- package/docs-site/blog/2021-08-01-mdx-blog-post.mdx +24 -0
- package/docs-site/blog/2021-08-26-welcome/docusaurus-plushie-banner.jpeg +0 -0
- package/docs-site/blog/2021-08-26-welcome/index.md +29 -0
- package/docs-site/blog/authors.yml +23 -0
- package/docs-site/blog/tags.yml +19 -0
- package/docs-site/docs/Architecture.md +76 -0
- package/docs-site/docs/CLI.md +69 -0
- package/docs-site/docs/Evaluation.md +61 -0
- package/docs-site/docs/FAQ.md +50 -0
- package/docs-site/docs/Introduction.md +58 -0
- package/docs-site/docs/Plugins.md +61 -0
- package/docs-site/docs/Usage.md +72 -0
- package/docs-site/docusaurus.config.js +57 -0
- package/docs-site/package-lock.json +17914 -0
- package/docs-site/package.json +44 -0
- package/docs-site/sidebars.js +22 -0
- package/docs-site/src/components/HomepageFeatures/index.js +59 -0
- package/docs-site/src/components/HomepageFeatures/styles.module.css +58 -0
- package/docs-site/src/css/custom.css +81 -0
- package/docs-site/src/pages/index.js +37 -0
- package/docs-site/src/pages/index.module.css +23 -0
- package/docs-site/src/pages/markdown-page.md +7 -0
- package/docs-site/static/.nojekyll +0 -0
- package/docs-site/static/img/favicon.ico +0 -0
- package/docs-site/static/img/logo.png +0 -0
- package/docs-site/static/img/logo.svg +1539 -0
- package/docs-site/static/img/social-card.jpg +0 -0
- package/jest.config.js +14 -0
- package/package.json +65 -0
- package/public/index.html +0 -0
- package/public/results/sample-eval.json +0 -0
- package/public/src/App.jsx +57 -0
- package/release.config.js +23 -0
- package/server.js +31 -0
- package/src/config/load-config.js +35 -0
- package/src/config/validate-schema.js +49 -0
- package/src/core/create-pipeline.js +82 -0
- package/src/core/plugin-registry.js +61 -0
- package/src/dag/dag-engine.js +70 -0
- package/src/evaluate/evaluator.js +44 -0
- package/src/evaluate/scoring.js +74 -0
- package/src/loader/csv-loader.js +30 -0
- package/src/loader/directory-loader.js +38 -0
- package/src/loader/html-loader.js +57 -0
- package/src/loader/markdown-loader.js +58 -0
- package/src/reranker/llm-reranker.js +49 -0
- package/src/utils/logger.js +29 -0
- package/src/utils/retry.js +42 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "docs-site",
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"private": true,
|
|
5
|
+
"scripts": {
|
|
6
|
+
"docusaurus": "docusaurus",
|
|
7
|
+
"start": "docusaurus start",
|
|
8
|
+
"build": "docusaurus build",
|
|
9
|
+
"swizzle": "docusaurus swizzle",
|
|
10
|
+
"deploy": "docusaurus deploy",
|
|
11
|
+
"clear": "docusaurus clear",
|
|
12
|
+
"serve": "docusaurus serve",
|
|
13
|
+
"write-translations": "docusaurus write-translations",
|
|
14
|
+
"write-heading-ids": "docusaurus write-heading-ids"
|
|
15
|
+
},
|
|
16
|
+
"dependencies": {
|
|
17
|
+
"@docusaurus/core": "3.7.0",
|
|
18
|
+
"@docusaurus/preset-classic": "3.7.0",
|
|
19
|
+
"@mdx-js/react": "^3.0.0",
|
|
20
|
+
"clsx": "^2.0.0",
|
|
21
|
+
"prism-react-renderer": "^2.3.0",
|
|
22
|
+
"react": "^19.0.0",
|
|
23
|
+
"react-dom": "^19.0.0"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"@docusaurus/module-type-aliases": "3.7.0",
|
|
27
|
+
"@docusaurus/types": "3.7.0"
|
|
28
|
+
},
|
|
29
|
+
"browserslist": {
|
|
30
|
+
"production": [
|
|
31
|
+
">0.5%",
|
|
32
|
+
"not dead",
|
|
33
|
+
"not op_mini all"
|
|
34
|
+
],
|
|
35
|
+
"development": [
|
|
36
|
+
"last 3 chrome version",
|
|
37
|
+
"last 3 firefox version",
|
|
38
|
+
"last 5 safari version"
|
|
39
|
+
]
|
|
40
|
+
},
|
|
41
|
+
"engines": {
|
|
42
|
+
"node": ">=18.0"
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Sidebar configuration for RAG Pipeline Utils documentation site
|
|
5
|
+
* Generated manually for ordered navigation
|
|
6
|
+
* Author: Ali Kahwaji
|
|
7
|
+
* @type {import('@docusaurus/plugin-content-docs').SidebarsConfig}
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const sidebars = {
|
|
11
|
+
tutorialSidebar: [
|
|
12
|
+
'Introduction',
|
|
13
|
+
'Usage',
|
|
14
|
+
'CLI',
|
|
15
|
+
'Architecture',
|
|
16
|
+
'Evaluation',
|
|
17
|
+
'Plugins',
|
|
18
|
+
'FAQ'
|
|
19
|
+
]
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export default sidebars;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import clsx from 'clsx';
|
|
3
|
+
import styles from './styles.module.css';
|
|
4
|
+
|
|
5
|
+
import FeatureImage from '@site/static/img/logo.svg';
|
|
6
|
+
|
|
7
|
+
const FeatureList = [
|
|
8
|
+
{
|
|
9
|
+
title: 'Composable Pipelines',
|
|
10
|
+
Svg: FeatureImage,
|
|
11
|
+
description: (
|
|
12
|
+
<>
|
|
13
|
+
Build flexible RAG workflows with interchangeable loaders, embedders, retrievers, and LLM runners using a clean plugin-based design.
|
|
14
|
+
</>
|
|
15
|
+
),
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
title: 'Evaluation & Dashboard',
|
|
19
|
+
Svg: FeatureImage,
|
|
20
|
+
description: (
|
|
21
|
+
<>
|
|
22
|
+
Use built-in CLI tooling and React dashboards to assess LLM accuracy, BLEU/ROUGE scores, and prompt results at scale.
|
|
23
|
+
</>
|
|
24
|
+
),
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
title: 'Fully Extendable',
|
|
28
|
+
Svg: FeatureImage,
|
|
29
|
+
description: (
|
|
30
|
+
<>
|
|
31
|
+
Register custom components with the PluginRegistry and extend the system without modifying the core logic.
|
|
32
|
+
</>
|
|
33
|
+
),
|
|
34
|
+
},
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
function Feature({Svg, title, description}) {
|
|
38
|
+
return (
|
|
39
|
+
<div className={clsx('col col--4', styles.feature)}>
|
|
40
|
+
<div className="text--center">
|
|
41
|
+
<Svg className={styles.featureSvg} role="img" />
|
|
42
|
+
</div>
|
|
43
|
+
<div className="text--center padding-horiz--md">
|
|
44
|
+
<h3 className={styles.featureTitle}>{title}</h3>
|
|
45
|
+
<p className={styles.featureText}>{description}</p>
|
|
46
|
+
</div>
|
|
47
|
+
</div>
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export default function HomepageFeatures() {
|
|
52
|
+
return (
|
|
53
|
+
<section className={styles.features}>
|
|
54
|
+
{FeatureList.map((props, idx) => (
|
|
55
|
+
<Feature key={idx} {...props} />
|
|
56
|
+
))}
|
|
57
|
+
</section>
|
|
58
|
+
);
|
|
59
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Version: 0.2.0
|
|
3
|
+
* Path: /docs-site/src/components/HomepageFeatures/styles.module.css
|
|
4
|
+
* Description: Modular styles for homepage feature section
|
|
5
|
+
* Author: Ali Kahwaji
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
.features {
|
|
9
|
+
display: flex;
|
|
10
|
+
flex-direction: row;
|
|
11
|
+
align-items: flex-start;
|
|
12
|
+
justify-content: space-around;
|
|
13
|
+
flex-wrap: wrap;
|
|
14
|
+
padding: 3rem 1rem;
|
|
15
|
+
gap: 2rem;
|
|
16
|
+
width: 100%;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
.feature {
|
|
20
|
+
flex: 1 1 300px;
|
|
21
|
+
max-width: 400px;
|
|
22
|
+
padding: 1rem;
|
|
23
|
+
text-align: center;
|
|
24
|
+
border: 1px solid #eaeaea;
|
|
25
|
+
border-radius: 8px;
|
|
26
|
+
transition: box-shadow 0.3s ease;
|
|
27
|
+
background-color: #f9f9f9;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
.feature:hover {
|
|
31
|
+
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
.featureSvg {
|
|
35
|
+
height: 180px;
|
|
36
|
+
width: 180px;
|
|
37
|
+
margin-bottom: 1rem;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
.featureTitle {
|
|
41
|
+
font-size: 1.25rem;
|
|
42
|
+
font-weight: 600;
|
|
43
|
+
margin-bottom: 0.5rem;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
.featureText {
|
|
47
|
+
font-size: 1rem;
|
|
48
|
+
color: #666;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
[data-theme='dark'] .feature {
|
|
52
|
+
background-color: #1c1e21;
|
|
53
|
+
border-color: #333;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
[data-theme='dark'] .featureText {
|
|
57
|
+
color: #ccc;
|
|
58
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Version: 0.2.0
|
|
3
|
+
* Path: /docs-site/src/css/custom.css
|
|
4
|
+
* Description: Customized global styles for RAG Pipeline Utils Docusaurus site
|
|
5
|
+
* Author: Ali Kahwaji
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/* Infima overrides */
|
|
9
|
+
:root {
|
|
10
|
+
--ifm-color-primary: #2e8555;
|
|
11
|
+
--ifm-color-primary-dark: #29784c;
|
|
12
|
+
--ifm-color-primary-darker: #277148;
|
|
13
|
+
--ifm-color-primary-darkest: #205d3b;
|
|
14
|
+
--ifm-color-primary-light: #33925d;
|
|
15
|
+
--ifm-color-primary-lighter: #359962;
|
|
16
|
+
--ifm-color-primary-lightest: #3cad6e;
|
|
17
|
+
--ifm-code-font-size: 95%;
|
|
18
|
+
--docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.05);
|
|
19
|
+
|
|
20
|
+
/* Custom enhancements */
|
|
21
|
+
--ifm-font-family-base: 'Segoe UI', Roboto, sans-serif;
|
|
22
|
+
--ifm-spacing-horizontal: 1.2rem;
|
|
23
|
+
--ifm-container-width: 90%;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
[data-theme='dark'] {
|
|
27
|
+
--ifm-color-primary: #25c2a0;
|
|
28
|
+
--ifm-color-primary-dark: #21af90;
|
|
29
|
+
--ifm-color-primary-darker: #1fa588;
|
|
30
|
+
--ifm-color-primary-darkest: #1a8870;
|
|
31
|
+
--ifm-color-primary-light: #29d5b0;
|
|
32
|
+
--ifm-color-primary-lighter: #32d8b4;
|
|
33
|
+
--ifm-color-primary-lightest: #4fddbf;
|
|
34
|
+
--docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/* Hero section styling */
|
|
38
|
+
main.container {
|
|
39
|
+
padding-top: 3rem;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
.hero__subtitle {
|
|
43
|
+
font-size: 1.25rem;
|
|
44
|
+
color: #444;
|
|
45
|
+
margin-bottom: 2rem;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
[data-theme='dark'] .hero__subtitle {
|
|
49
|
+
color: #ccc;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/* Button enhancements */
|
|
53
|
+
.button--info {
|
|
54
|
+
background-color: #e0f2ff !important;
|
|
55
|
+
color: #0366d6 !important;
|
|
56
|
+
border: 1px solid #b6e0fe !important;
|
|
57
|
+
font-weight: 500;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
.button--info:hover {
|
|
61
|
+
background-color: #d2ecfc !important;
|
|
62
|
+
border-color: #a4d7fa !important;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/* Footer padding */
|
|
66
|
+
.footer {
|
|
67
|
+
padding-top: 2rem;
|
|
68
|
+
padding-bottom: 2rem;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/* Improve readability of inline code */
|
|
72
|
+
code {
|
|
73
|
+
background-color: rgba(27,31,35,.05);
|
|
74
|
+
padding: 0.2em 0.4em;
|
|
75
|
+
border-radius: 4px;
|
|
76
|
+
font-size: 90%;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
[data-theme='dark'] code {
|
|
80
|
+
background-color: rgba(255,255,255,0.08);
|
|
81
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import Layout from '@theme/Layout';
|
|
3
|
+
import Link from '@docusaurus/Link';
|
|
4
|
+
|
|
5
|
+
export default function Home() {
|
|
6
|
+
return (
|
|
7
|
+
<Layout title="RAG Pipeline Utils" description="Modular tools for retrieval-augmented generation">
|
|
8
|
+
<main className="container margin-vert--xl text--center">
|
|
9
|
+
<h1>Welcome to RAG Pipeline Utils</h1>
|
|
10
|
+
<p className="hero__subtitle">
|
|
11
|
+
A composable, pluggable RAG framework for Node.js with CLI, evaluation, and dashboard support.
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<div style={{ marginTop: '2rem' }}>
|
|
15
|
+
<Link
|
|
16
|
+
className="button button--primary button--lg margin-horiz--sm"
|
|
17
|
+
to="/docs/Introduction">
|
|
18
|
+
📄 Get Started
|
|
19
|
+
</Link>
|
|
20
|
+
|
|
21
|
+
<Link
|
|
22
|
+
className="button button--secondary button--lg margin-horiz--sm"
|
|
23
|
+
to="/docs/CLI">
|
|
24
|
+
🧰 CLI Reference
|
|
25
|
+
</Link>
|
|
26
|
+
|
|
27
|
+
<Link
|
|
28
|
+
className="button button--info button--lg margin-horiz--sm"
|
|
29
|
+
style={{ backgroundColor: '#e0f2ff', color: '#0366d6', border: '1px solid #b6e0fe' }}
|
|
30
|
+
to="/docs/Evaluation">
|
|
31
|
+
📈 Evaluation Dashboard
|
|
32
|
+
</Link>
|
|
33
|
+
</div>
|
|
34
|
+
</main>
|
|
35
|
+
</Layout>
|
|
36
|
+
);
|
|
37
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSS files with the .module.css suffix will be treated as CSS modules
|
|
3
|
+
* and scoped locally.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
.heroBanner {
|
|
7
|
+
padding: 4rem 0;
|
|
8
|
+
text-align: center;
|
|
9
|
+
position: relative;
|
|
10
|
+
overflow: hidden;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
@media screen and (max-width: 996px) {
|
|
14
|
+
.heroBanner {
|
|
15
|
+
padding: 2rem;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
.buttons {
|
|
20
|
+
display: flex;
|
|
21
|
+
align-items: center;
|
|
22
|
+
justify-content: center;
|
|
23
|
+
}
|
|
File without changes
|
|
Binary file
|
|
Binary file
|