@vivantel/virage-strategies 0.2.15 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -7
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -22,8 +22,14 @@ import {
|
|
|
22
22
|
} from "@vivantel/virage-strategies";
|
|
23
23
|
|
|
24
24
|
const chunkers = [
|
|
25
|
-
createChunker({
|
|
26
|
-
|
|
25
|
+
createChunker({
|
|
26
|
+
patterns: ["docs/**/*.md"],
|
|
27
|
+
strategy: markdownHeadersStrategy(),
|
|
28
|
+
}),
|
|
29
|
+
createChunker({
|
|
30
|
+
patterns: ["src/**/*.ts"],
|
|
31
|
+
strategy: tokenStrategy({ maxTokens: 500 }),
|
|
32
|
+
}),
|
|
27
33
|
createChunker({ patterns: ["**/*.yaml"], strategy: wholeFileStrategy() }),
|
|
28
34
|
];
|
|
29
35
|
```
|
|
@@ -34,7 +40,11 @@ Or via JSON config (no TypeScript needed):
|
|
|
34
40
|
{
|
|
35
41
|
"chunkers": [
|
|
36
42
|
{ "patterns": ["docs/**/*.md"], "strategy": "markdownHeaders" },
|
|
37
|
-
{
|
|
43
|
+
{
|
|
44
|
+
"patterns": ["src/**/*.ts"],
|
|
45
|
+
"strategy": "token",
|
|
46
|
+
"strategyOptions": { "maxTokens": 500 }
|
|
47
|
+
}
|
|
38
48
|
]
|
|
39
49
|
}
|
|
40
50
|
```
|
|
@@ -51,10 +61,10 @@ Best for: documentation, wikis, README files.
|
|
|
51
61
|
|
|
52
62
|
Splits text into fixed-size token windows with optional overlap.
|
|
53
63
|
|
|
54
|
-
| Option
|
|
55
|
-
|
|
|
56
|
-
| `maxTokens` | `512`
|
|
57
|
-
| `overlap`
|
|
64
|
+
| Option | Default | Description |
|
|
65
|
+
| ----------- | ------- | ---------------------------------------- |
|
|
66
|
+
| `maxTokens` | `512` | Maximum tokens per chunk |
|
|
67
|
+
| `overlap` | `50` | Token overlap between consecutive chunks |
|
|
58
68
|
|
|
59
69
|
Best for: source code, structured text, anything that needs size control.
|
|
60
70
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vivantel/virage-strategies",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.17",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Built-in chunking strategies for @vivantel/rag-core",
|
|
6
6
|
"repository": {
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"@vivantel/virage-core": "*"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@vivantel/virage-core": "0.2.
|
|
50
|
+
"@vivantel/virage-core": "0.2.16",
|
|
51
51
|
"@types/node": "^25.9.1",
|
|
52
52
|
"typescript": "^6.0.3",
|
|
53
53
|
"vitest": "^4.1.8"
|