@vivantel/rag-core 0.1.0 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. package/.github/config/release-please.json +37 -37
  2. package/.github/dependabot.yaml +27 -27
  3. package/.github/workflows/ci.yaml +119 -119
  4. package/.github/workflows/publish.yaml +155 -151
  5. package/.github/workflows/release.yaml +54 -150
  6. package/.release-please-manifest.json +3 -0
  7. package/.versionrc.json +18 -18
  8. package/CHANGELOG.md +51 -21
  9. package/README.md +62 -62
  10. package/bin/rag-update.ts +48 -48
  11. package/eslint.config.js +24 -24
  12. package/package.json +106 -102
  13. package/tsconfig.json +20 -20
  14. package/typedoc.json +10 -10
  15. package/vitest.config.ts +18 -18
package/README.md CHANGED
@@ -1,62 +1,62 @@
1
- # @vivantel/rag-core
2
-
3
- Core RAG pipeline tools - universal chunking, embedding, and vector store orchestration.
4
-
5
- [![CI](https://github.com/vivantel/rag-core/actions/workflows/ci.yaml/badge.svg)](https://github.com/vivantel/rag-core/actions/workflows/ci.yaml)
6
- [![Release](https://github.com/vivantel/rag-core/actions/workflows/release.yaml/badge.svg)](https://github.com/vivantel/rag-core/actions/workflows/release.yaml)
7
- [![npm version](https://img.shields.io/npm/v/@vivantel/rag-core.svg)](https://www.npmjs.com/package/@vivantel/rag-core)
8
- [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
9
-
10
- ## Installation
11
-
12
- ```bash
13
- npm install @vivantel/rag-core
14
- ```
15
-
16
- ## Quick Start
17
-
18
- ### 1. Create a config file (`rag.config.ts`)
19
-
20
- ```typescript
21
- import { defineConfig, createChunker, tokenStrategy } from '@vivantel/rag-core';
22
- import { readFile } from 'fs/promises';
23
-
24
- export default defineConfig({
25
- chunkers: [
26
- createChunker({
27
- name: 'my-chunker',
28
- patterns: ['docs/**/*.md'],
29
- async process(content, filePath, commitHash) {
30
- return [{
31
- content,
32
- metadata: { type: 'doc', file: filePath },
33
- sourceFile: filePath,
34
- commitHash
35
- }];
36
- }
37
- })
38
- ],
39
-
40
- embedder: new MyEmbedderProvider({ apiKey: process.env.API_KEY }),
41
- vectorStore: new MyVectorStore({ url: process.env.DB_URL })
42
- });
43
- ```
44
-
45
- ### 2. Run the pipeline
46
-
47
- ```bash
48
- rag-update --config rag.config.ts
49
- ```
50
-
51
- ## Built-in Strategies
52
-
53
- ### Chunk Strategies
54
-
55
- - `tokenStrategy({ maxTokens, overlap })` - Split by approximate token count
56
- - `markdownHeadersStrategy()` - Split by Markdown headers (##, ###)
57
- - `semanticStrategy()` - Split by sentence boundaries
58
- - `wholeFileStrategy()` - One chunk per file
59
-
60
- ## License
61
-
62
- MIT
1
+ # @vivantel/rag-core
2
+
3
+ Core RAG pipeline tools - universal chunking, embedding, and vector store orchestration.
4
+
5
+ [![CI](https://github.com/vivantel/rag-core/actions/workflows/ci.yaml/badge.svg)](https://github.com/vivantel/rag-core/actions/workflows/ci.yaml)
6
+ [![Release](https://github.com/vivantel/rag-core/actions/workflows/release.yaml/badge.svg)](https://github.com/vivantel/rag-core/actions/workflows/release.yaml)
7
+ [![npm version](https://img.shields.io/npm/v/@vivantel/rag-core.svg)](https://www.npmjs.com/package/@vivantel/rag-core)
8
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
9
+
10
+ ## Installation
11
+
12
+ ```bash
13
+ npm install @vivantel/rag-core
14
+ ```
15
+
16
+ ## Quick Start
17
+
18
+ ### 1. Create a config file (`rag.config.ts`)
19
+
20
+ ```typescript
21
+ import { defineConfig, createChunker, tokenStrategy } from '@vivantel/rag-core';
22
+ import { readFile } from 'fs/promises';
23
+
24
+ export default defineConfig({
25
+ chunkers: [
26
+ createChunker({
27
+ name: 'my-chunker',
28
+ patterns: ['docs/**/*.md'],
29
+ async process(content, filePath, commitHash) {
30
+ return [{
31
+ content,
32
+ metadata: { type: 'doc', file: filePath },
33
+ sourceFile: filePath,
34
+ commitHash
35
+ }];
36
+ }
37
+ })
38
+ ],
39
+
40
+ embedder: new MyEmbedderProvider({ apiKey: process.env.API_KEY }),
41
+ vectorStore: new MyVectorStore({ url: process.env.DB_URL })
42
+ });
43
+ ```
44
+
45
+ ### 2. Run the pipeline
46
+
47
+ ```bash
48
+ rag-update --config rag.config.ts
49
+ ```
50
+
51
+ ## Built-in Strategies
52
+
53
+ ### Chunk Strategies
54
+
55
+ - `tokenStrategy({ maxTokens, overlap })` - Split by approximate token count
56
+ - `markdownHeadersStrategy()` - Split by Markdown headers (##, ###)
57
+ - `semanticStrategy()` - Split by sentence boundaries
58
+ - `wholeFileStrategy()` - One chunk per file
59
+
60
+ ## License
61
+
62
+ MIT
package/bin/rag-update.ts CHANGED
@@ -1,49 +1,49 @@
1
- #!/usr/bin/env node
2
-
3
- import { Command } from 'commander';
4
- import { config } from 'dotenv';
5
- import { loadConfig } from '../src/config-loader.js';
6
- import { Orchestrator } from '../src/core/orchestrator.js';
7
-
8
- config();
9
-
10
- const program = new Command();
11
-
12
- program
13
- .name('rag-update')
14
- .description('Update RAG index with latest changes')
15
- .version('1.0.0')
16
- .option('-c, --config <path>', 'Path to config file', './rag.config.ts')
17
- .option('-f, --force', 'Force full rebuild', false)
18
- .option('--skip-upload', 'Skip upload to vector store', false)
19
- .option('--chunks-file <path>', 'Output path for chunks.json')
20
- .option('--embeddings-file <path>', 'Output path for embeddings.json')
21
- .parse();
22
-
23
- async function main() {
24
- const options = program.opts();
25
-
26
- console.log('🚀 RAG Update Tool\n');
27
-
28
- try {
29
- const config = await loadConfig(options.config);
30
-
31
- const orchestrator = new Orchestrator({
32
- ...config,
33
- options: {
34
- ...config.options,
35
- force: options.force || config.options?.force,
36
- skipUpload: options.skipUpload || config.options?.skipUpload,
37
- chunksFile: options.chunksFile || config.options?.chunksFile,
38
- embeddingsFile: options.embeddingsFile || config.options?.embeddingsFile
39
- }
40
- });
41
-
42
- await orchestrator.run();
43
- } catch (error) {
44
- console.error('❌ Error:', error instanceof Error ? error.message : error);
45
- process.exit(1);
46
- }
47
- }
48
-
1
+ #!/usr/bin/env node
2
+
3
+ import { Command } from 'commander';
4
+ import { config } from 'dotenv';
5
+ import { loadConfig } from '../src/config-loader.js';
6
+ import { Orchestrator } from '../src/core/orchestrator.js';
7
+
8
+ config();
9
+
10
+ const program = new Command();
11
+
12
+ program
13
+ .name('rag-update')
14
+ .description('Update RAG index with latest changes')
15
+ .version('1.0.0')
16
+ .option('-c, --config <path>', 'Path to config file', './rag.config.ts')
17
+ .option('-f, --force', 'Force full rebuild', false)
18
+ .option('--skip-upload', 'Skip upload to vector store', false)
19
+ .option('--chunks-file <path>', 'Output path for chunks.json')
20
+ .option('--embeddings-file <path>', 'Output path for embeddings.json')
21
+ .parse();
22
+
23
+ async function main() {
24
+ const options = program.opts();
25
+
26
+ console.log('🚀 RAG Update Tool\n');
27
+
28
+ try {
29
+ const config = await loadConfig(options.config);
30
+
31
+ const orchestrator = new Orchestrator({
32
+ ...config,
33
+ options: {
34
+ ...config.options,
35
+ force: options.force || config.options?.force,
36
+ skipUpload: options.skipUpload || config.options?.skipUpload,
37
+ chunksFile: options.chunksFile || config.options?.chunksFile,
38
+ embeddingsFile: options.embeddingsFile || config.options?.embeddingsFile
39
+ }
40
+ });
41
+
42
+ await orchestrator.run();
43
+ } catch (error) {
44
+ console.error('❌ Error:', error instanceof Error ? error.message : error);
45
+ process.exit(1);
46
+ }
47
+ }
48
+
49
49
  main();
package/eslint.config.js CHANGED
@@ -1,25 +1,25 @@
1
- import js from '@eslint/js';
2
- import tseslint from 'typescript-eslint';
3
-
4
- export default tseslint.config(
5
- js.configs.recommended,
6
- ...tseslint.configs.recommended,
7
- {
8
- ignores: [
9
- 'dist/**',
10
- 'node_modules/**',
11
- 'coverage/**',
12
- '*.config.js',
13
- '*.config.ts'
14
- ]
15
- },
16
- {
17
- files: ['src/**/*.ts'],
18
- rules: {
19
- '@typescript-eslint/no-explicit-any': 'warn',
20
- '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
21
- 'no-console': 'off',
22
- 'no-undef': 'off'
23
- }
24
- }
1
+ import js from '@eslint/js';
2
+ import tseslint from 'typescript-eslint';
3
+
4
+ export default tseslint.config(
5
+ js.configs.recommended,
6
+ ...tseslint.configs.recommended,
7
+ {
8
+ ignores: [
9
+ 'dist/**',
10
+ 'node_modules/**',
11
+ 'coverage/**',
12
+ '*.config.js',
13
+ '*.config.ts'
14
+ ]
15
+ },
16
+ {
17
+ files: ['src/**/*.ts'],
18
+ rules: {
19
+ '@typescript-eslint/no-explicit-any': 'warn',
20
+ '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
21
+ 'no-console': 'off',
22
+ 'no-undef': 'off'
23
+ }
24
+ }
25
25
  );
package/package.json CHANGED
@@ -1,102 +1,106 @@
1
- {
2
- "name": "@vivantel/rag-core",
3
- "version": "0.1.0",
4
- "type": "module",
5
- "description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
6
- "main": "dist/index.js",
7
- "types": "dist/index.d.ts",
8
- "bin": {
9
- "rag-update": "./dist/bin/rag-update.js"
10
- },
11
- "scripts": {
12
- "audit:fix": "npm audit fix",
13
- "audit:report": "npm audit --json > npm-audit-report.json",
14
- "audit": "npm audit",
15
- "build:clean": "rm -rf dist && npm run build",
16
- "build:verbose": "tsc --listEmittedFiles",
17
- "build": "tsc && echo \"✅ Build completed successfully\"",
18
- "check:full": "npm run check && npm run test:coverage && npm run type-coverage",
19
- "check:quick": "npm run type-check && npm run lint",
20
- "check": "npm run type-check && npm run lint && npm run format:check && npm run audit",
21
- "clean:cache": "npm cache clean --force",
22
- "clean:rebuild": "npm run clean && npm install && npm run build",
23
- "clean": "rm -rf dist node_modules coverage docs",
24
- "dev:debug": "tsx --inspect-brk src/index.ts",
25
- "dev": "tsx watch src/index.ts",
26
- "docs:generate": "typedoc",
27
- "docs:open": "npm run docs:generate && open docs/index.html",
28
- "docs:serve": "npm run docs:generate && npx http-server docs -p 8080",
29
- "fix:aggressive": "npm run fix:all && npm run update:apply && npm run clean:rebuild",
30
- "fix:all": "npm run fix && npm run audit:fix",
31
- "fix": "npm run lint:fix && npm run format",
32
- "format:check": "prettier --check \"src/**/*.ts\"",
33
- "format": "prettier --write \"src/**/*.ts\"",
34
- "help": "echo 'Available commands:' && grep -E '^\\s+\"[a-z]' package.json | sed 's/\"//g' | sed 's/://g' | column -t",
35
- "info": "echo 'Node: $(node --version) | npm: $(npm --version)' && npm list --depth=0",
36
- "lint:fix": "eslint src/ --fix",
37
- "lint:strict": "eslint src/ --max-warnings 0",
38
- "lint": "eslint src/",
39
- "outdated:report": "npm outdated --json > npm-outdated-report.json",
40
- "outdated": "npm outdated",
41
- "postpublish": "echo \"✅ Published to npm\"",
42
- "postversion": "git push && git push --tags",
43
- "precommit": "npm run type-check && npm run format:check && npm run lint && npm audit",
44
- "prepublishOnly": "npm run build && npm test",
45
- "prepush": "npm run test:run && npm run audit && npm run format:check",
46
- "release:major": "npm version major && npm publish",
47
- "release:minor": "npm version minor && npm publish",
48
- "release:patch": "npm version patch && npm publish",
49
- "release:pre": "npm version prerelease && npm publish --tag next",
50
- "status": "npm run outdated && npm run audit",
51
- "test:coverage": "vitest --coverage --config vitest.config.ts",
52
- "test:debug": "vitest --inspect-brk --config vitest.config.ts",
53
- "test:run": "vitest run --config vitest.config.ts",
54
- "test:update": "vitest -u --config vitest.config.ts",
55
- "test:watch": "vitest watch --config vitest.config.ts",
56
- "test": "vitest run --config vitest.config.ts",
57
- "type-check:watch": "tsc --noEmit --watch",
58
- "type-check": "tsc --noEmit",
59
- "type-coverage:report": "type-coverage --detail",
60
- "type-coverage": "type-coverage --at-least 90",
61
- "update:apply": "npx npm-check-updates -u && npm install",
62
- "update:check": "npx npm-check-updates",
63
- "update": "npm update",
64
- "version": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md",
65
- "watch:test": "vitest --watch --config vitest.config.ts",
66
- "watch": "tsc --watch"
67
- },
68
- "keywords": [
69
- "rag",
70
- "embeddings",
71
- "vector",
72
- "git",
73
- "pipeline"
74
- ],
75
- "author": "Vivantel",
76
- "license": "MIT",
77
- "dependencies": {
78
- "commander": "^15.0.0",
79
- "dotenv": "^17.4.2",
80
- "glob": "^13.0.6",
81
- "simple-git": "^3.25.0"
82
- },
83
- "devDependencies": {
84
- "@eslint/js": "^10.0.1",
85
- "@types/node": "^25.9.1",
86
- "eslint": "^10.4.1",
87
- "prettier": "^3.8.3",
88
- "tsx": "^4.11.0",
89
- "typedoc": "^0.28.19",
90
- "typedoc-plugin-markdown": "^4.11.0",
91
- "typescript": "^6.0.3",
92
- "typescript-eslint": "^8.60.0",
93
- "vitest": "^4.1.7"
94
- },
95
- "engines": {
96
- "node": ">=18.0.0"
97
- },
98
- "vitest": {
99
- "environment": "node",
100
- "globals": false
101
- }
102
- }
1
+ {
2
+ "name": "@vivantel/rag-core",
3
+ "version": "1.1.2",
4
+ "type": "module",
5
+ "repository": {
6
+ "type": "git",
7
+ "url": "https://github.com/vivantel/rag_core"
8
+ },
9
+ "description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
10
+ "main": "dist/index.js",
11
+ "types": "dist/index.d.ts",
12
+ "bin": {
13
+ "rag-update": "./dist/bin/rag-update.js"
14
+ },
15
+ "scripts": {
16
+ "audit:fix": "npm audit fix",
17
+ "audit:report": "npm audit --json > npm-audit-report.json",
18
+ "audit": "npm audit",
19
+ "build:clean": "rm -rf dist && npm run build",
20
+ "build:verbose": "tsc --listEmittedFiles",
21
+ "build": "tsc && echo \"✅ Build completed successfully\"",
22
+ "check:full": "npm run check && npm run test:coverage && npm run type-coverage",
23
+ "check:quick": "npm run type-check && npm run lint",
24
+ "check": "npm run type-check && npm run lint && npm run format:check && npm run audit",
25
+ "clean:cache": "npm cache clean --force",
26
+ "clean:rebuild": "npm run clean && npm install && npm run build",
27
+ "clean": "rm -rf dist node_modules coverage docs",
28
+ "dev:debug": "tsx --inspect-brk src/index.ts",
29
+ "dev": "tsx watch src/index.ts",
30
+ "docs:generate": "typedoc",
31
+ "docs:open": "npm run docs:generate && open docs/index.html",
32
+ "docs:serve": "npm run docs:generate && npx http-server docs -p 8080",
33
+ "fix:aggressive": "npm run fix:all && npm run update:apply && npm run clean:rebuild",
34
+ "fix:all": "npm run fix && npm run audit:fix",
35
+ "fix": "npm run lint:fix && npm run format",
36
+ "format:check": "prettier --check \"src/**/*.ts\"",
37
+ "format": "prettier --write \"src/**/*.ts\"",
38
+ "help": "echo 'Available commands:' && grep -E '^\\s+\"[a-z]' package.json | sed 's/\"//g' | sed 's/://g' | column -t",
39
+ "info": "echo 'Node: $(node --version) | npm: $(npm --version)' && npm list --depth=0",
40
+ "lint:fix": "eslint src/ --fix",
41
+ "lint:strict": "eslint src/ --max-warnings 0",
42
+ "lint": "eslint src/",
43
+ "outdated:report": "npm outdated --json > npm-outdated-report.json",
44
+ "outdated": "npm outdated",
45
+ "postpublish": "echo \"✅ Published to npm\"",
46
+ "postversion": "git push && git push --tags",
47
+ "precommit": "npm run type-check && npm run format:check && npm run lint && npm audit",
48
+ "prepublishOnly": "npm run build && npm test",
49
+ "prepush": "npm run test:run && npm run audit && npm run format:check",
50
+ "release:major": "npm version major && npm publish",
51
+ "release:minor": "npm version minor && npm publish",
52
+ "release:patch": "npm version patch && npm publish",
53
+ "release:pre": "npm version prerelease && npm publish --tag next",
54
+ "status": "npm run outdated && npm run audit",
55
+ "test:coverage": "vitest --coverage --config vitest.config.ts",
56
+ "test:debug": "vitest --inspect-brk --config vitest.config.ts",
57
+ "test:run": "vitest run --config vitest.config.ts",
58
+ "test:update": "vitest -u --config vitest.config.ts",
59
+ "test:watch": "vitest watch --config vitest.config.ts",
60
+ "test": "vitest run --config vitest.config.ts",
61
+ "type-check:watch": "tsc --noEmit --watch",
62
+ "type-check": "tsc --noEmit",
63
+ "type-coverage:report": "type-coverage --detail",
64
+ "type-coverage": "type-coverage --at-least 90",
65
+ "update:apply": "npx npm-check-updates -u && npm install",
66
+ "update:check": "npx npm-check-updates",
67
+ "update": "npm update",
68
+ "version": "conventional-changelog -p angular -i CHANGELOG.md -s && git add CHANGELOG.md",
69
+ "watch:test": "vitest --watch --config vitest.config.ts",
70
+ "watch": "tsc --watch"
71
+ },
72
+ "keywords": [
73
+ "rag",
74
+ "embeddings",
75
+ "vector",
76
+ "git",
77
+ "pipeline"
78
+ ],
79
+ "author": "Vivantel",
80
+ "license": "MIT",
81
+ "dependencies": {
82
+ "commander": "^15.0.0",
83
+ "dotenv": "^17.4.2",
84
+ "glob": "^13.0.6",
85
+ "simple-git": "^3.25.0"
86
+ },
87
+ "devDependencies": {
88
+ "@eslint/js": "^10.0.1",
89
+ "@types/node": "^25.9.1",
90
+ "eslint": "^10.4.1",
91
+ "prettier": "^3.8.3",
92
+ "tsx": "^4.11.0",
93
+ "typedoc": "^0.28.19",
94
+ "typedoc-plugin-markdown": "^4.11.0",
95
+ "typescript": "^6.0.3",
96
+ "typescript-eslint": "^8.60.0",
97
+ "vitest": "^4.1.7"
98
+ },
99
+ "engines": {
100
+ "node": ">=18.0.0"
101
+ },
102
+ "vitest": {
103
+ "environment": "node",
104
+ "globals": false
105
+ }
106
+ }
package/tsconfig.json CHANGED
@@ -1,21 +1,21 @@
1
- {
2
- "compilerOptions": {
3
- "target": "ES2022",
4
- "module": "NodeNext",
5
- "moduleResolution": "NodeNext",
6
- "lib": ["ES2022"],
7
- "outDir": "./dist",
8
- "rootDir": "./src",
9
- "declaration": true,
10
- "declarationMap": true,
11
- "sourceMap": true,
12
- "strict": true,
13
- "esModuleInterop": true,
14
- "skipLibCheck": true,
15
- "forceConsistentCasingInFileNames": true,
16
- "resolveJsonModule": true,
17
- "types": ["node"]
18
- },
19
- "include": ["src/**/*"],
20
- "exclude": ["node_modules", "dist", "**/*.test.ts"]
1
+ {
2
+ "compilerOptions": {
3
+ "target": "ES2022",
4
+ "module": "NodeNext",
5
+ "moduleResolution": "NodeNext",
6
+ "lib": ["ES2022"],
7
+ "outDir": "./dist",
8
+ "rootDir": "./src",
9
+ "declaration": true,
10
+ "declarationMap": true,
11
+ "sourceMap": true,
12
+ "strict": true,
13
+ "esModuleInterop": true,
14
+ "skipLibCheck": true,
15
+ "forceConsistentCasingInFileNames": true,
16
+ "resolveJsonModule": true,
17
+ "types": ["node"]
18
+ },
19
+ "include": ["src/**/*"],
20
+ "exclude": ["node_modules", "dist", "**/*.test.ts"]
21
21
  }
package/typedoc.json CHANGED
@@ -1,11 +1,11 @@
1
- {
2
- "entryPoints": ["src/index.ts"],
3
- "out": "docs/api",
4
- "excludePrivate": true,
5
- "excludeProtected": true,
6
- "skipErrorChecking": true,
7
- "validation": {
8
- "invalidLink": false,
9
- "notExported": false
10
- }
1
+ {
2
+ "entryPoints": ["src/index.ts"],
3
+ "out": "docs/api",
4
+ "excludePrivate": true,
5
+ "excludeProtected": true,
6
+ "skipErrorChecking": true,
7
+ "validation": {
8
+ "invalidLink": false,
9
+ "notExported": false
10
+ }
11
11
  }
package/vitest.config.ts CHANGED
@@ -1,19 +1,19 @@
1
- import { defineConfig } from 'vitest/config';
2
-
3
- export default defineConfig({
4
- test: {
5
- maxWorkers: 4,
6
- isolate: false,
7
-
8
- coverage: {
9
- provider: 'v8',
10
- reporter: ['text', 'json', 'html'],
11
- include: ['src/**/*.ts'],
12
- exclude: ['src/**/*.test.ts'],
13
- },
14
- exclude: ['node_modules', '.git'],
15
- globals: true,
16
- testTimeout: 10000,
17
- environment: 'node',
18
- },
1
+ import { defineConfig } from 'vitest/config';
2
+
3
+ export default defineConfig({
4
+ test: {
5
+ maxWorkers: 4,
6
+ isolate: false,
7
+
8
+ coverage: {
9
+ provider: 'v8',
10
+ reporter: ['text', 'json', 'html'],
11
+ include: ['src/**/*.ts'],
12
+ exclude: ['src/**/*.test.ts'],
13
+ },
14
+ exclude: ['node_modules', '.git'],
15
+ globals: true,
16
+ testTimeout: 10000,
17
+ environment: 'node',
18
+ },
19
19
  });