@vibe-agent-toolkit/rag 0.1.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +424 -0
- package/dist/chunking/chunk-by-tokens.d.ts +22 -0
- package/dist/chunking/chunk-by-tokens.d.ts.map +1 -0
- package/dist/chunking/chunk-by-tokens.js +68 -0
- package/dist/chunking/chunk-by-tokens.js.map +1 -0
- package/dist/chunking/chunk-resource.d.ts +46 -0
- package/dist/chunking/chunk-resource.d.ts.map +1 -0
- package/dist/chunking/chunk-resource.js +131 -0
- package/dist/chunking/chunk-resource.js.map +1 -0
- package/dist/chunking/index.d.ts +10 -0
- package/dist/chunking/index.d.ts.map +1 -0
- package/dist/chunking/index.js +9 -0
- package/dist/chunking/index.js.map +1 -0
- package/dist/chunking/types.d.ts +42 -0
- package/dist/chunking/types.d.ts.map +1 -0
- package/dist/chunking/types.js +5 -0
- package/dist/chunking/types.js.map +1 -0
- package/dist/chunking/utils.d.ts +41 -0
- package/dist/chunking/utils.d.ts.map +1 -0
- package/dist/chunking/utils.js +62 -0
- package/dist/chunking/utils.js.map +1 -0
- package/dist/embedding-providers/index.d.ts +8 -0
- package/dist/embedding-providers/index.d.ts.map +1 -0
- package/dist/embedding-providers/index.js +8 -0
- package/dist/embedding-providers/index.js.map +1 -0
- package/dist/embedding-providers/openai-embedding-provider.d.ts +64 -0
- package/dist/embedding-providers/openai-embedding-provider.d.ts.map +1 -0
- package/dist/embedding-providers/openai-embedding-provider.js +92 -0
- package/dist/embedding-providers/openai-embedding-provider.js.map +1 -0
- package/dist/embedding-providers/transformers-embedding-provider.d.ts +62 -0
- package/dist/embedding-providers/transformers-embedding-provider.d.ts.map +1 -0
- package/dist/embedding-providers/transformers-embedding-provider.js +75 -0
- package/dist/embedding-providers/transformers-embedding-provider.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/interfaces/embedding.d.ts +33 -0
- package/dist/interfaces/embedding.d.ts.map +1 -0
- package/dist/interfaces/embedding.js +7 -0
- package/dist/interfaces/embedding.js.map +1 -0
- package/dist/interfaces/index.d.ts +9 -0
- package/dist/interfaces/index.d.ts.map +1 -0
- package/dist/interfaces/index.js +7 -0
- package/dist/interfaces/index.js.map +1 -0
- package/dist/interfaces/provider.d.ts +150 -0
- package/dist/interfaces/provider.d.ts.map +1 -0
- package/dist/interfaces/provider.js +8 -0
- package/dist/interfaces/provider.js.map +1 -0
- package/dist/interfaces/token-counter.d.ts +29 -0
- package/dist/interfaces/token-counter.d.ts.map +1 -0
- package/dist/interfaces/token-counter.js +7 -0
- package/dist/interfaces/token-counter.js.map +1 -0
- package/dist/schemas/admin.d.ts +82 -0
- package/dist/schemas/admin.d.ts.map +1 -0
- package/dist/schemas/admin.js +34 -0
- package/dist/schemas/admin.js.map +1 -0
- package/dist/schemas/chunk.d.ts +75 -0
- package/dist/schemas/chunk.d.ts.map +1 -0
- package/dist/schemas/chunk.js +39 -0
- package/dist/schemas/chunk.js.map +1 -0
- package/dist/schemas/index.d.ts +9 -0
- package/dist/schemas/index.d.ts.map +1 -0
- package/dist/schemas/index.js +9 -0
- package/dist/schemas/index.js.map +1 -0
- package/dist/schemas/json-schema.d.ts +86 -0
- package/dist/schemas/json-schema.d.ts.map +1 -0
- package/dist/schemas/json-schema.js +55 -0
- package/dist/schemas/json-schema.js.map +1 -0
- package/dist/schemas/query.d.ts +262 -0
- package/dist/schemas/query.d.ts.map +1 -0
- package/dist/schemas/query.js +56 -0
- package/dist/schemas/query.js.map +1 -0
- package/dist/token-counters/approximate-token-counter.d.ts +32 -0
- package/dist/token-counters/approximate-token-counter.d.ts.map +1 -0
- package/dist/token-counters/approximate-token-counter.js +40 -0
- package/dist/token-counters/approximate-token-counter.js.map +1 -0
- package/dist/token-counters/fast-token-counter.d.ts +33 -0
- package/dist/token-counters/fast-token-counter.d.ts.map +1 -0
- package/dist/token-counters/fast-token-counter.js +40 -0
- package/dist/token-counters/fast-token-counter.js.map +1 -0
- package/dist/token-counters/index.d.ts +8 -0
- package/dist/token-counters/index.d.ts.map +1 -0
- package/dist/token-counters/index.js +8 -0
- package/dist/token-counters/index.js.map +1 -0
- package/package.json +53 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../src/schemas/chunk.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,WAAW;IACX,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;IAC9D,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAE3E,UAAU;IACV,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACrD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;IACxE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;IAEzE,oCAAoC;IACpC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qDAAqD,CAAC;IAClG,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iDAAiD,CAAC;IAC/F,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAChF,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAE5E,sGAAsG;IACtG,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IACjD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;IAC9D,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC;IACrD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAEvD,qBAAqB;IACrB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAC3D,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gEAAgE,CAAC;IACrG,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IAE7D,0BAA0B;IAC1B,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IAC3F,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sCAAsC,CAAC;CACpF,CAAC,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Schemas
|
|
3
|
+
*
|
|
4
|
+
* Zod schemas for RAG data structures with generated JSON schemas.
|
|
5
|
+
*/
|
|
6
|
+
export { RAGChunkSchema, type RAGChunk } from './chunk.js';
|
|
7
|
+
export { RAGQuerySchema, RAGResultSchema, type RAGQuery, type RAGResult } from './query.js';
|
|
8
|
+
export { RAGStatsSchema, IndexResultSchema, type RAGStats, type IndexResult } from './admin.js';
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schemas/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAE,KAAK,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,KAAK,QAAQ,EAAE,KAAK,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,KAAK,QAAQ,EAAE,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Schemas
|
|
3
|
+
*
|
|
4
|
+
* Zod schemas for RAG data structures with generated JSON schemas.
|
|
5
|
+
*/
|
|
6
|
+
export { RAGChunkSchema } from './chunk.js';
|
|
7
|
+
export { RAGQuerySchema, RAGResultSchema } from './query.js';
|
|
8
|
+
export { RAGStatsSchema, IndexResultSchema } from './admin.js';
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/schemas/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,cAAc,EAAiB,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAiC,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAmC,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON Schema generation from Zod schemas
|
|
3
|
+
*
|
|
4
|
+
* Exports JSON schemas for use by external tools and documentation.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Generate JSON Schema for RAGChunk
|
|
8
|
+
*/
|
|
9
|
+
export declare const RAGChunkJsonSchema: import("zod-to-json-schema").JsonSchema7Type & {
|
|
10
|
+
$schema?: string | undefined;
|
|
11
|
+
definitions?: {
|
|
12
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
13
|
+
} | undefined;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Generate JSON Schema for RAGQuery
|
|
17
|
+
*/
|
|
18
|
+
export declare const RAGQueryJsonSchema: import("zod-to-json-schema").JsonSchema7Type & {
|
|
19
|
+
$schema?: string | undefined;
|
|
20
|
+
definitions?: {
|
|
21
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
22
|
+
} | undefined;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Generate JSON Schema for RAGResult
|
|
26
|
+
*/
|
|
27
|
+
export declare const RAGResultJsonSchema: import("zod-to-json-schema").JsonSchema7Type & {
|
|
28
|
+
$schema?: string | undefined;
|
|
29
|
+
definitions?: {
|
|
30
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
31
|
+
} | undefined;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Generate JSON Schema for RAGStats
|
|
35
|
+
*/
|
|
36
|
+
export declare const RAGStatsJsonSchema: import("zod-to-json-schema").JsonSchema7Type & {
|
|
37
|
+
$schema?: string | undefined;
|
|
38
|
+
definitions?: {
|
|
39
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
40
|
+
} | undefined;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Generate JSON Schema for IndexResult
|
|
44
|
+
*/
|
|
45
|
+
export declare const IndexResultJsonSchema: import("zod-to-json-schema").JsonSchema7Type & {
|
|
46
|
+
$schema?: string | undefined;
|
|
47
|
+
definitions?: {
|
|
48
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
49
|
+
} | undefined;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* All JSON Schemas exported as a collection
|
|
53
|
+
*/
|
|
54
|
+
export declare const jsonSchemas: {
|
|
55
|
+
RAGChunk: import("zod-to-json-schema").JsonSchema7Type & {
|
|
56
|
+
$schema?: string | undefined;
|
|
57
|
+
definitions?: {
|
|
58
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
59
|
+
} | undefined;
|
|
60
|
+
};
|
|
61
|
+
RAGQuery: import("zod-to-json-schema").JsonSchema7Type & {
|
|
62
|
+
$schema?: string | undefined;
|
|
63
|
+
definitions?: {
|
|
64
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
65
|
+
} | undefined;
|
|
66
|
+
};
|
|
67
|
+
RAGResult: import("zod-to-json-schema").JsonSchema7Type & {
|
|
68
|
+
$schema?: string | undefined;
|
|
69
|
+
definitions?: {
|
|
70
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
71
|
+
} | undefined;
|
|
72
|
+
};
|
|
73
|
+
RAGStats: import("zod-to-json-schema").JsonSchema7Type & {
|
|
74
|
+
$schema?: string | undefined;
|
|
75
|
+
definitions?: {
|
|
76
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
77
|
+
} | undefined;
|
|
78
|
+
};
|
|
79
|
+
IndexResult: import("zod-to-json-schema").JsonSchema7Type & {
|
|
80
|
+
$schema?: string | undefined;
|
|
81
|
+
definitions?: {
|
|
82
|
+
[key: string]: import("zod-to-json-schema").JsonSchema7Type;
|
|
83
|
+
} | undefined;
|
|
84
|
+
};
|
|
85
|
+
};
|
|
86
|
+
//# sourceMappingURL=json-schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-schema.d.ts","sourceRoot":"","sources":["../../src/schemas/json-schema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH;;GAEG;AACH,eAAO,MAAM,kBAAkB;;;;;CAG7B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,kBAAkB;;;;;CAG7B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,mBAAmB;;;;;CAG9B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,kBAAkB;;;;;CAG7B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,qBAAqB;;;;;CAGhC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAMvB,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON Schema generation from Zod schemas
|
|
3
|
+
*
|
|
4
|
+
* Exports JSON schemas for use by external tools and documentation.
|
|
5
|
+
*/
|
|
6
|
+
import { zodToJsonSchema } from 'zod-to-json-schema';
|
|
7
|
+
import { IndexResultSchema, RAGStatsSchema } from './admin.js';
|
|
8
|
+
import { RAGChunkSchema } from './chunk.js';
|
|
9
|
+
import { RAGQuerySchema, RAGResultSchema } from './query.js';
|
|
10
|
+
/**
|
|
11
|
+
* Generate JSON Schema for RAGChunk
|
|
12
|
+
*/
|
|
13
|
+
export const RAGChunkJsonSchema = zodToJsonSchema(RAGChunkSchema, {
|
|
14
|
+
name: 'RAGChunk',
|
|
15
|
+
$refStrategy: 'none',
|
|
16
|
+
});
|
|
17
|
+
/**
|
|
18
|
+
* Generate JSON Schema for RAGQuery
|
|
19
|
+
*/
|
|
20
|
+
export const RAGQueryJsonSchema = zodToJsonSchema(RAGQuerySchema, {
|
|
21
|
+
name: 'RAGQuery',
|
|
22
|
+
$refStrategy: 'none',
|
|
23
|
+
});
|
|
24
|
+
/**
|
|
25
|
+
* Generate JSON Schema for RAGResult
|
|
26
|
+
*/
|
|
27
|
+
export const RAGResultJsonSchema = zodToJsonSchema(RAGResultSchema, {
|
|
28
|
+
name: 'RAGResult',
|
|
29
|
+
$refStrategy: 'none',
|
|
30
|
+
});
|
|
31
|
+
/**
|
|
32
|
+
* Generate JSON Schema for RAGStats
|
|
33
|
+
*/
|
|
34
|
+
export const RAGStatsJsonSchema = zodToJsonSchema(RAGStatsSchema, {
|
|
35
|
+
name: 'RAGStats',
|
|
36
|
+
$refStrategy: 'none',
|
|
37
|
+
});
|
|
38
|
+
/**
|
|
39
|
+
* Generate JSON Schema for IndexResult
|
|
40
|
+
*/
|
|
41
|
+
export const IndexResultJsonSchema = zodToJsonSchema(IndexResultSchema, {
|
|
42
|
+
name: 'IndexResult',
|
|
43
|
+
$refStrategy: 'none',
|
|
44
|
+
});
|
|
45
|
+
/**
|
|
46
|
+
* All JSON Schemas exported as a collection
|
|
47
|
+
*/
|
|
48
|
+
export const jsonSchemas = {
|
|
49
|
+
RAGChunk: RAGChunkJsonSchema,
|
|
50
|
+
RAGQuery: RAGQueryJsonSchema,
|
|
51
|
+
RAGResult: RAGResultJsonSchema,
|
|
52
|
+
RAGStats: RAGStatsJsonSchema,
|
|
53
|
+
IndexResult: IndexResultJsonSchema,
|
|
54
|
+
};
|
|
55
|
+
//# sourceMappingURL=json-schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json-schema.js","sourceRoot":"","sources":["../../src/schemas/json-schema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAE7D;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC,cAAc,EAAE;IAChE,IAAI,EAAE,UAAU;IAChB,YAAY,EAAE,MAAM;CACrB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC,cAAc,EAAE;IAChE,IAAI,EAAE,UAAU;IAChB,YAAY,EAAE,MAAM;CACrB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,eAAe,CAAC,eAAe,EAAE;IAClE,IAAI,EAAE,WAAW;IACjB,YAAY,EAAE,MAAM;CACrB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC,cAAc,EAAE;IAChE,IAAI,EAAE,UAAU;IAChB,YAAY,EAAE,MAAM;CACrB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,eAAe,CAAC,iBAAiB,EAAE;IACtE,IAAI,EAAE,aAAa;IACnB,YAAY,EAAE,MAAM;CACrB,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,QAAQ,EAAE,kBAAkB;IAC5B,QAAQ,EAAE,kBAAkB;IAC5B,SAAS,EAAE,mBAAmB;IAC9B,QAAQ,EAAE,kBAAkB;IAC5B,WAAW,EAAE,qBAAqB;CACnC,CAAC"}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAGQuery and RAGResult Zod schemas
|
|
3
|
+
*/
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
/**
|
|
6
|
+
* RAGQuery Schema
|
|
7
|
+
*
|
|
8
|
+
* Defines the structure of a query to the RAG database.
|
|
9
|
+
*/
|
|
10
|
+
export declare const RAGQuerySchema: z.ZodObject<{
|
|
11
|
+
/** Search query text */
|
|
12
|
+
text: z.ZodString;
|
|
13
|
+
/** Maximum results to return (default: 10) */
|
|
14
|
+
limit: z.ZodOptional<z.ZodNumber>;
|
|
15
|
+
/** Metadata filters */
|
|
16
|
+
filters: z.ZodOptional<z.ZodObject<{
|
|
17
|
+
/** Filter by resource ID(s) */
|
|
18
|
+
resourceId: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>;
|
|
19
|
+
/** Filter by tags */
|
|
20
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
21
|
+
/** Filter by resource type */
|
|
22
|
+
type: z.ZodOptional<z.ZodString>;
|
|
23
|
+
/** Filter by heading path */
|
|
24
|
+
headingPath: z.ZodOptional<z.ZodString>;
|
|
25
|
+
/** Filter by date range */
|
|
26
|
+
dateRange: z.ZodOptional<z.ZodObject<{
|
|
27
|
+
start: z.ZodDate;
|
|
28
|
+
end: z.ZodDate;
|
|
29
|
+
}, "strip", z.ZodTypeAny, {
|
|
30
|
+
start: Date;
|
|
31
|
+
end: Date;
|
|
32
|
+
}, {
|
|
33
|
+
start: Date;
|
|
34
|
+
end: Date;
|
|
35
|
+
}>>;
|
|
36
|
+
}, "strip", z.ZodTypeAny, {
|
|
37
|
+
resourceId?: string | string[] | undefined;
|
|
38
|
+
headingPath?: string | undefined;
|
|
39
|
+
type?: string | undefined;
|
|
40
|
+
tags?: string[] | undefined;
|
|
41
|
+
dateRange?: {
|
|
42
|
+
start: Date;
|
|
43
|
+
end: Date;
|
|
44
|
+
} | undefined;
|
|
45
|
+
}, {
|
|
46
|
+
resourceId?: string | string[] | undefined;
|
|
47
|
+
headingPath?: string | undefined;
|
|
48
|
+
type?: string | undefined;
|
|
49
|
+
tags?: string[] | undefined;
|
|
50
|
+
dateRange?: {
|
|
51
|
+
start: Date;
|
|
52
|
+
end: Date;
|
|
53
|
+
} | undefined;
|
|
54
|
+
}>>;
|
|
55
|
+
/** Hybrid search configuration */
|
|
56
|
+
hybridSearch: z.ZodOptional<z.ZodObject<{
|
|
57
|
+
enabled: z.ZodBoolean;
|
|
58
|
+
keywordWeight: z.ZodOptional<z.ZodNumber>;
|
|
59
|
+
}, "strip", z.ZodTypeAny, {
|
|
60
|
+
enabled: boolean;
|
|
61
|
+
keywordWeight?: number | undefined;
|
|
62
|
+
}, {
|
|
63
|
+
enabled: boolean;
|
|
64
|
+
keywordWeight?: number | undefined;
|
|
65
|
+
}>>;
|
|
66
|
+
}, "strip", z.ZodTypeAny, {
|
|
67
|
+
text: string;
|
|
68
|
+
limit?: number | undefined;
|
|
69
|
+
filters?: {
|
|
70
|
+
resourceId?: string | string[] | undefined;
|
|
71
|
+
headingPath?: string | undefined;
|
|
72
|
+
type?: string | undefined;
|
|
73
|
+
tags?: string[] | undefined;
|
|
74
|
+
dateRange?: {
|
|
75
|
+
start: Date;
|
|
76
|
+
end: Date;
|
|
77
|
+
} | undefined;
|
|
78
|
+
} | undefined;
|
|
79
|
+
hybridSearch?: {
|
|
80
|
+
enabled: boolean;
|
|
81
|
+
keywordWeight?: number | undefined;
|
|
82
|
+
} | undefined;
|
|
83
|
+
}, {
|
|
84
|
+
text: string;
|
|
85
|
+
limit?: number | undefined;
|
|
86
|
+
filters?: {
|
|
87
|
+
resourceId?: string | string[] | undefined;
|
|
88
|
+
headingPath?: string | undefined;
|
|
89
|
+
type?: string | undefined;
|
|
90
|
+
tags?: string[] | undefined;
|
|
91
|
+
dateRange?: {
|
|
92
|
+
start: Date;
|
|
93
|
+
end: Date;
|
|
94
|
+
} | undefined;
|
|
95
|
+
} | undefined;
|
|
96
|
+
hybridSearch?: {
|
|
97
|
+
enabled: boolean;
|
|
98
|
+
keywordWeight?: number | undefined;
|
|
99
|
+
} | undefined;
|
|
100
|
+
}>;
|
|
101
|
+
/**
|
|
102
|
+
* RAGQuery TypeScript type
|
|
103
|
+
*/
|
|
104
|
+
export type RAGQuery = z.infer<typeof RAGQuerySchema>;
|
|
105
|
+
/**
|
|
106
|
+
* RAGResult Schema
|
|
107
|
+
*
|
|
108
|
+
* Defines the structure of results from a RAG query.
|
|
109
|
+
*/
|
|
110
|
+
export declare const RAGResultSchema: z.ZodObject<{
|
|
111
|
+
/** Matched chunks, sorted by relevance */
|
|
112
|
+
chunks: z.ZodArray<z.ZodObject<{
|
|
113
|
+
chunkId: z.ZodString;
|
|
114
|
+
resourceId: z.ZodString;
|
|
115
|
+
content: z.ZodString;
|
|
116
|
+
contentHash: z.ZodString;
|
|
117
|
+
tokenCount: z.ZodNumber;
|
|
118
|
+
headingPath: z.ZodOptional<z.ZodString>;
|
|
119
|
+
headingLevel: z.ZodOptional<z.ZodNumber>;
|
|
120
|
+
startLine: z.ZodOptional<z.ZodNumber>;
|
|
121
|
+
endLine: z.ZodOptional<z.ZodNumber>;
|
|
122
|
+
filePath: z.ZodString;
|
|
123
|
+
tags: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
124
|
+
type: z.ZodOptional<z.ZodString>;
|
|
125
|
+
title: z.ZodOptional<z.ZodString>;
|
|
126
|
+
embedding: z.ZodArray<z.ZodNumber, "many">;
|
|
127
|
+
embeddingModel: z.ZodString;
|
|
128
|
+
embeddedAt: z.ZodDate;
|
|
129
|
+
previousChunkId: z.ZodOptional<z.ZodString>;
|
|
130
|
+
nextChunkId: z.ZodOptional<z.ZodString>;
|
|
131
|
+
}, "strip", z.ZodTypeAny, {
|
|
132
|
+
chunkId: string;
|
|
133
|
+
resourceId: string;
|
|
134
|
+
content: string;
|
|
135
|
+
contentHash: string;
|
|
136
|
+
tokenCount: number;
|
|
137
|
+
filePath: string;
|
|
138
|
+
embedding: number[];
|
|
139
|
+
embeddingModel: string;
|
|
140
|
+
embeddedAt: Date;
|
|
141
|
+
headingPath?: string | undefined;
|
|
142
|
+
headingLevel?: number | undefined;
|
|
143
|
+
startLine?: number | undefined;
|
|
144
|
+
endLine?: number | undefined;
|
|
145
|
+
type?: string | undefined;
|
|
146
|
+
tags?: string[] | undefined;
|
|
147
|
+
title?: string | undefined;
|
|
148
|
+
previousChunkId?: string | undefined;
|
|
149
|
+
nextChunkId?: string | undefined;
|
|
150
|
+
}, {
|
|
151
|
+
chunkId: string;
|
|
152
|
+
resourceId: string;
|
|
153
|
+
content: string;
|
|
154
|
+
contentHash: string;
|
|
155
|
+
tokenCount: number;
|
|
156
|
+
filePath: string;
|
|
157
|
+
embedding: number[];
|
|
158
|
+
embeddingModel: string;
|
|
159
|
+
embeddedAt: Date;
|
|
160
|
+
headingPath?: string | undefined;
|
|
161
|
+
headingLevel?: number | undefined;
|
|
162
|
+
startLine?: number | undefined;
|
|
163
|
+
endLine?: number | undefined;
|
|
164
|
+
type?: string | undefined;
|
|
165
|
+
tags?: string[] | undefined;
|
|
166
|
+
title?: string | undefined;
|
|
167
|
+
previousChunkId?: string | undefined;
|
|
168
|
+
nextChunkId?: string | undefined;
|
|
169
|
+
}>, "many">;
|
|
170
|
+
/** Search statistics */
|
|
171
|
+
stats: z.ZodObject<{
|
|
172
|
+
totalMatches: z.ZodNumber;
|
|
173
|
+
searchDurationMs: z.ZodNumber;
|
|
174
|
+
embedding: z.ZodOptional<z.ZodObject<{
|
|
175
|
+
model: z.ZodString;
|
|
176
|
+
tokensUsed: z.ZodOptional<z.ZodNumber>;
|
|
177
|
+
}, "strip", z.ZodTypeAny, {
|
|
178
|
+
model: string;
|
|
179
|
+
tokensUsed?: number | undefined;
|
|
180
|
+
}, {
|
|
181
|
+
model: string;
|
|
182
|
+
tokensUsed?: number | undefined;
|
|
183
|
+
}>>;
|
|
184
|
+
}, "strip", z.ZodTypeAny, {
|
|
185
|
+
totalMatches: number;
|
|
186
|
+
searchDurationMs: number;
|
|
187
|
+
embedding?: {
|
|
188
|
+
model: string;
|
|
189
|
+
tokensUsed?: number | undefined;
|
|
190
|
+
} | undefined;
|
|
191
|
+
}, {
|
|
192
|
+
totalMatches: number;
|
|
193
|
+
searchDurationMs: number;
|
|
194
|
+
embedding?: {
|
|
195
|
+
model: string;
|
|
196
|
+
tokensUsed?: number | undefined;
|
|
197
|
+
} | undefined;
|
|
198
|
+
}>;
|
|
199
|
+
}, "strip", z.ZodTypeAny, {
|
|
200
|
+
chunks: {
|
|
201
|
+
chunkId: string;
|
|
202
|
+
resourceId: string;
|
|
203
|
+
content: string;
|
|
204
|
+
contentHash: string;
|
|
205
|
+
tokenCount: number;
|
|
206
|
+
filePath: string;
|
|
207
|
+
embedding: number[];
|
|
208
|
+
embeddingModel: string;
|
|
209
|
+
embeddedAt: Date;
|
|
210
|
+
headingPath?: string | undefined;
|
|
211
|
+
headingLevel?: number | undefined;
|
|
212
|
+
startLine?: number | undefined;
|
|
213
|
+
endLine?: number | undefined;
|
|
214
|
+
type?: string | undefined;
|
|
215
|
+
tags?: string[] | undefined;
|
|
216
|
+
title?: string | undefined;
|
|
217
|
+
previousChunkId?: string | undefined;
|
|
218
|
+
nextChunkId?: string | undefined;
|
|
219
|
+
}[];
|
|
220
|
+
stats: {
|
|
221
|
+
totalMatches: number;
|
|
222
|
+
searchDurationMs: number;
|
|
223
|
+
embedding?: {
|
|
224
|
+
model: string;
|
|
225
|
+
tokensUsed?: number | undefined;
|
|
226
|
+
} | undefined;
|
|
227
|
+
};
|
|
228
|
+
}, {
|
|
229
|
+
chunks: {
|
|
230
|
+
chunkId: string;
|
|
231
|
+
resourceId: string;
|
|
232
|
+
content: string;
|
|
233
|
+
contentHash: string;
|
|
234
|
+
tokenCount: number;
|
|
235
|
+
filePath: string;
|
|
236
|
+
embedding: number[];
|
|
237
|
+
embeddingModel: string;
|
|
238
|
+
embeddedAt: Date;
|
|
239
|
+
headingPath?: string | undefined;
|
|
240
|
+
headingLevel?: number | undefined;
|
|
241
|
+
startLine?: number | undefined;
|
|
242
|
+
endLine?: number | undefined;
|
|
243
|
+
type?: string | undefined;
|
|
244
|
+
tags?: string[] | undefined;
|
|
245
|
+
title?: string | undefined;
|
|
246
|
+
previousChunkId?: string | undefined;
|
|
247
|
+
nextChunkId?: string | undefined;
|
|
248
|
+
}[];
|
|
249
|
+
stats: {
|
|
250
|
+
totalMatches: number;
|
|
251
|
+
searchDurationMs: number;
|
|
252
|
+
embedding?: {
|
|
253
|
+
model: string;
|
|
254
|
+
tokensUsed?: number | undefined;
|
|
255
|
+
} | undefined;
|
|
256
|
+
};
|
|
257
|
+
}>;
|
|
258
|
+
/**
|
|
259
|
+
* RAGResult TypeScript type
|
|
260
|
+
*/
|
|
261
|
+
export type RAGResult = z.infer<typeof RAGResultSchema>;
|
|
262
|
+
//# sourceMappingURL=query.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query.d.ts","sourceRoot":"","sources":["../../src/schemas/query.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;GAIG;AACH,eAAO,MAAM,cAAc;IACzB,wBAAwB;;IAGxB,8CAA8C;;IAG9C,uBAAuB;;QAErB,+BAA+B;;QAE/B,qBAAqB;;QAErB,8BAA8B;;QAE9B,6BAA6B;;QAE7B,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAO7B,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKlC,CAAC;AAEH;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD;;;;GAIG;AACH,eAAO,MAAM,eAAe;IAC1B,0CAA0C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAG1C,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASxB,CAAC;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAGQuery and RAGResult Zod schemas
|
|
3
|
+
*/
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { RAGChunkSchema } from './chunk.js';
|
|
6
|
+
/**
|
|
7
|
+
* RAGQuery Schema
|
|
8
|
+
*
|
|
9
|
+
* Defines the structure of a query to the RAG database.
|
|
10
|
+
*/
|
|
11
|
+
export const RAGQuerySchema = z.object({
|
|
12
|
+
/** Search query text */
|
|
13
|
+
text: z.string().describe('Search query text'),
|
|
14
|
+
/** Maximum results to return (default: 10) */
|
|
15
|
+
limit: z.number().optional().describe('Maximum results to return'),
|
|
16
|
+
/** Metadata filters */
|
|
17
|
+
filters: z.object({
|
|
18
|
+
/** Filter by resource ID(s) */
|
|
19
|
+
resourceId: z.union([z.string(), z.array(z.string())]).optional().describe('Filter by resource ID(s)'),
|
|
20
|
+
/** Filter by tags */
|
|
21
|
+
tags: z.array(z.string()).optional().describe('Filter by tags'),
|
|
22
|
+
/** Filter by resource type */
|
|
23
|
+
type: z.string().optional().describe('Filter by resource type'),
|
|
24
|
+
/** Filter by heading path */
|
|
25
|
+
headingPath: z.string().optional().describe('Filter by heading path (e.g., "Architecture > RAG Design")'),
|
|
26
|
+
/** Filter by date range */
|
|
27
|
+
dateRange: z.object({
|
|
28
|
+
start: z.date(),
|
|
29
|
+
end: z.date(),
|
|
30
|
+
}).optional().describe('Filter by date range'),
|
|
31
|
+
}).optional().describe('Metadata filters'),
|
|
32
|
+
/** Hybrid search configuration */
|
|
33
|
+
hybridSearch: z.object({
|
|
34
|
+
enabled: z.boolean().describe('Enable hybrid search (vector + keyword)'),
|
|
35
|
+
keywordWeight: z.number().optional().describe('Keyword weight (0-1, balance between semantic and keyword)'),
|
|
36
|
+
}).optional().describe('Hybrid search configuration'),
|
|
37
|
+
});
|
|
38
|
+
/**
|
|
39
|
+
* RAGResult Schema
|
|
40
|
+
*
|
|
41
|
+
* Defines the structure of results from a RAG query.
|
|
42
|
+
*/
|
|
43
|
+
export const RAGResultSchema = z.object({
|
|
44
|
+
/** Matched chunks, sorted by relevance */
|
|
45
|
+
chunks: z.array(RAGChunkSchema).describe('Matched chunks, sorted by relevance'),
|
|
46
|
+
/** Search statistics */
|
|
47
|
+
stats: z.object({
|
|
48
|
+
totalMatches: z.number().describe('Total number of matches'),
|
|
49
|
+
searchDurationMs: z.number().describe('Search duration in milliseconds'),
|
|
50
|
+
embedding: z.object({
|
|
51
|
+
model: z.string().describe('Embedding model used'),
|
|
52
|
+
tokensUsed: z.number().optional().describe('Tokens used for embedding (if applicable)'),
|
|
53
|
+
}).optional().describe('Embedding statistics'),
|
|
54
|
+
}).describe('Search statistics'),
|
|
55
|
+
});
|
|
56
|
+
//# sourceMappingURL=query.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/schemas/query.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,wBAAwB;IACxB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAE9C,8CAA8C;IAC9C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IAElE,uBAAuB;IACvB,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,+BAA+B;QAC/B,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC;QACtG,qBAAqB;QACrB,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QAC/D,8BAA8B;QAC9B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAC/D,6BAA6B;QAC7B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4DAA4D,CAAC;QACzG,2BAA2B;QAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC;YAClB,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE;YACf,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE;SACd,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;KAC/C,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kBAAkB,CAAC;IAE1C,kCAAkC;IAClC,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC;QACrB,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;QACxE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4DAA4D,CAAC;KAC5G,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;CACtD,CAAC,CAAC;AAOH;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,0CAA0C;IAC1C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAE/E,wBAAwB;IACxB,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;QACd,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAC5D,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;QACxE,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC;YAClB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;YAClD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;SACxF,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;KAC/C,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC;CACjC,CAAC,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Approximate Token Counter
|
|
3
|
+
*
|
|
4
|
+
* Uses gpt-tokenizer for accurate token counting compatible with GPT-3.5/GPT-4.
|
|
5
|
+
*/
|
|
6
|
+
import type { TokenCounter } from '../interfaces/token-counter.js';
|
|
7
|
+
/**
|
|
8
|
+
* ApproximateTokenCounter
|
|
9
|
+
*
|
|
10
|
+
* Uses gpt-tokenizer library for accurate token counting.
|
|
11
|
+
* Compatible with GPT-3.5/GPT-4 tokenization (cl100k_base encoding).
|
|
12
|
+
*
|
|
13
|
+
* Recommended padding factor: 0.9 (90% of target)
|
|
14
|
+
*/
|
|
15
|
+
export declare class ApproximateTokenCounter implements TokenCounter {
|
|
16
|
+
readonly name = "approximate";
|
|
17
|
+
/**
|
|
18
|
+
* Count tokens using gpt-tokenizer
|
|
19
|
+
*
|
|
20
|
+
* @param text - Text to count tokens in
|
|
21
|
+
* @returns Accurate token count
|
|
22
|
+
*/
|
|
23
|
+
count(text: string): number;
|
|
24
|
+
/**
|
|
25
|
+
* Count tokens in multiple texts
|
|
26
|
+
*
|
|
27
|
+
* @param texts - Array of texts to count tokens in
|
|
28
|
+
* @returns Array of token counts
|
|
29
|
+
*/
|
|
30
|
+
countBatch(texts: string[]): number[];
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=approximate-token-counter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"approximate-token-counter.d.ts","sourceRoot":"","sources":["../../src/token-counters/approximate-token-counter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAEnE;;;;;;;GAOG;AACH,qBAAa,uBAAwB,YAAW,YAAY;IAC1D,QAAQ,CAAC,IAAI,iBAAiB;IAE9B;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS3B;;;;;OAKG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;CAGtC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Approximate Token Counter
|
|
3
|
+
*
|
|
4
|
+
* Uses gpt-tokenizer for accurate token counting compatible with GPT-3.5/GPT-4.
|
|
5
|
+
*/
|
|
6
|
+
import { encode } from 'gpt-tokenizer';
|
|
7
|
+
/**
|
|
8
|
+
* ApproximateTokenCounter
|
|
9
|
+
*
|
|
10
|
+
* Uses gpt-tokenizer library for accurate token counting.
|
|
11
|
+
* Compatible with GPT-3.5/GPT-4 tokenization (cl100k_base encoding).
|
|
12
|
+
*
|
|
13
|
+
* Recommended padding factor: 0.9 (90% of target)
|
|
14
|
+
*/
|
|
15
|
+
export class ApproximateTokenCounter {
|
|
16
|
+
name = 'approximate';
|
|
17
|
+
/**
|
|
18
|
+
* Count tokens using gpt-tokenizer
|
|
19
|
+
*
|
|
20
|
+
* @param text - Text to count tokens in
|
|
21
|
+
* @returns Accurate token count
|
|
22
|
+
*/
|
|
23
|
+
count(text) {
|
|
24
|
+
if (text.length === 0) {
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
const tokens = encode(text);
|
|
28
|
+
return tokens.length;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Count tokens in multiple texts
|
|
32
|
+
*
|
|
33
|
+
* @param texts - Array of texts to count tokens in
|
|
34
|
+
* @returns Array of token counts
|
|
35
|
+
*/
|
|
36
|
+
countBatch(texts) {
|
|
37
|
+
return texts.map((text) => this.count(text));
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=approximate-token-counter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"approximate-token-counter.js","sourceRoot":"","sources":["../../src/token-counters/approximate-token-counter.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAIvC;;;;;;;GAOG;AACH,MAAM,OAAO,uBAAuB;IACzB,IAAI,GAAG,aAAa,CAAC;IAE9B;;;;;OAKG;IACH,KAAK,CAAC,IAAY;QAChB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,OAAO,MAAM,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACH,UAAU,CAAC,KAAe;QACxB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fast Token Counter
|
|
3
|
+
*
|
|
4
|
+
* Uses bytes/4 heuristic for token estimation.
|
|
5
|
+
* Fast but inaccurate - suitable for quick validation checks.
|
|
6
|
+
*/
|
|
7
|
+
import type { TokenCounter } from '../interfaces/token-counter.js';
|
|
8
|
+
/**
|
|
9
|
+
* FastTokenCounter
|
|
10
|
+
*
|
|
11
|
+
* Estimates token count using bytes/4 heuristic.
|
|
12
|
+
* Very fast, but accuracy is ~25% for English text.
|
|
13
|
+
*
|
|
14
|
+
* Recommended padding factor: 0.8 (80% of target)
|
|
15
|
+
*/
|
|
16
|
+
export declare class FastTokenCounter implements TokenCounter {
|
|
17
|
+
readonly name = "fast";
|
|
18
|
+
/**
|
|
19
|
+
* Count tokens using bytes/4 heuristic
|
|
20
|
+
*
|
|
21
|
+
* @param text - Text to count tokens in
|
|
22
|
+
* @returns Estimated token count
|
|
23
|
+
*/
|
|
24
|
+
count(text: string): number;
|
|
25
|
+
/**
|
|
26
|
+
* Count tokens in multiple texts
|
|
27
|
+
*
|
|
28
|
+
* @param texts - Array of texts to count tokens in
|
|
29
|
+
* @returns Array of estimated token counts
|
|
30
|
+
*/
|
|
31
|
+
countBatch(texts: string[]): number[];
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=fast-token-counter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fast-token-counter.d.ts","sourceRoot":"","sources":["../../src/token-counters/fast-token-counter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AAEnE;;;;;;;GAOG;AACH,qBAAa,gBAAiB,YAAW,YAAY;IACnD,QAAQ,CAAC,IAAI,UAAU;IAEvB;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS3B;;;;;OAKG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;CAGtC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fast Token Counter
|
|
3
|
+
*
|
|
4
|
+
* Uses bytes/4 heuristic for token estimation.
|
|
5
|
+
* Fast but inaccurate - suitable for quick validation checks.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* FastTokenCounter
|
|
9
|
+
*
|
|
10
|
+
* Estimates token count using bytes/4 heuristic.
|
|
11
|
+
* Very fast, but accuracy is ~25% for English text.
|
|
12
|
+
*
|
|
13
|
+
* Recommended padding factor: 0.8 (80% of target)
|
|
14
|
+
*/
|
|
15
|
+
export class FastTokenCounter {
|
|
16
|
+
name = 'fast';
|
|
17
|
+
/**
|
|
18
|
+
* Count tokens using bytes/4 heuristic
|
|
19
|
+
*
|
|
20
|
+
* @param text - Text to count tokens in
|
|
21
|
+
* @returns Estimated token count
|
|
22
|
+
*/
|
|
23
|
+
count(text) {
|
|
24
|
+
if (text.length === 0) {
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
const bytes = new TextEncoder().encode(text).length;
|
|
28
|
+
return Math.ceil(bytes / 4);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Count tokens in multiple texts
|
|
32
|
+
*
|
|
33
|
+
* @param texts - Array of texts to count tokens in
|
|
34
|
+
* @returns Array of estimated token counts
|
|
35
|
+
*/
|
|
36
|
+
countBatch(texts) {
|
|
37
|
+
return texts.map((text) => this.count(text));
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=fast-token-counter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fast-token-counter.js","sourceRoot":"","sources":["../../src/token-counters/fast-token-counter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;;;;;GAOG;AACH,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,MAAM,CAAC;IAEvB;;;;;OAKG;IACH,KAAK,CAAC,IAAY;QAChB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACpD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACH,UAAU,CAAC,KAAe;QACxB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|