@yolk-sdk/knowledge 0.1.0-canary.29 → 0.1.0-canary.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +52 -73
  2. package/dist/agent.d.mts +62 -16
  3. package/dist/agent.d.mts.map +1 -1
  4. package/dist/agent.mjs +172 -31
  5. package/dist/agent.mjs.map +1 -1
  6. package/dist/chunking.d.mts +2 -2
  7. package/dist/chunking.d.mts.map +1 -1
  8. package/dist/chunking.mjs +1 -1
  9. package/dist/chunking.mjs.map +1 -1
  10. package/dist/context.d.mts +3 -8
  11. package/dist/context.d.mts.map +1 -1
  12. package/dist/context.mjs +16 -11
  13. package/dist/context.mjs.map +1 -1
  14. package/dist/documents.d.mts +98 -57
  15. package/dist/documents.d.mts.map +1 -1
  16. package/dist/documents.mjs +83 -48
  17. package/dist/documents.mjs.map +1 -1
  18. package/dist/errors.d.mts +3 -3
  19. package/dist/errors.d.mts.map +1 -1
  20. package/dist/errors.mjs +11 -11
  21. package/dist/errors.mjs.map +1 -1
  22. package/dist/files.d.mts +22 -0
  23. package/dist/files.d.mts.map +1 -0
  24. package/dist/files.mjs +7 -0
  25. package/dist/files.mjs.map +1 -0
  26. package/dist/index.d.mts +4 -4
  27. package/dist/index.mjs +4 -4
  28. package/dist/ingestion.d.mts +8 -7
  29. package/dist/ingestion.d.mts.map +1 -1
  30. package/dist/ingestion.mjs +10 -14
  31. package/dist/ingestion.mjs.map +1 -1
  32. package/dist/search.d.mts +3 -3
  33. package/dist/search.d.mts.map +1 -1
  34. package/dist/search.mjs +4 -4
  35. package/dist/search.mjs.map +1 -1
  36. package/dist/store.d.mts +87 -21
  37. package/dist/store.d.mts.map +1 -1
  38. package/dist/store.mjs +2 -1
  39. package/dist/store.mjs.map +1 -1
  40. package/package.json +9 -39
  41. package/src/agent.ts +242 -68
  42. package/src/chunking.ts +2 -2
  43. package/src/context.ts +18 -16
  44. package/src/documents.ts +95 -64
  45. package/src/errors.ts +2 -2
  46. package/src/files.ts +24 -0
  47. package/src/index.ts +3 -3
  48. package/src/ingestion.ts +11 -16
  49. package/src/search.ts +6 -6
  50. package/src/store.ts +132 -24
  51. package/dist/artifacts.d.mts +0 -37
  52. package/dist/artifacts.d.mts.map +0 -1
  53. package/dist/artifacts.mjs +0 -28
  54. package/dist/artifacts.mjs.map +0 -1
  55. package/dist/links.d.mts +0 -17
  56. package/dist/links.d.mts.map +0 -1
  57. package/dist/links.mjs +0 -24
  58. package/dist/links.mjs.map +0 -1
  59. package/dist/provenance.d.mts +0 -20
  60. package/dist/provenance.d.mts.map +0 -1
  61. package/dist/provenance.mjs +0 -26
  62. package/dist/provenance.mjs.map +0 -1
  63. package/dist/records.d.mts +0 -59
  64. package/dist/records.d.mts.map +0 -1
  65. package/dist/records.mjs +0 -64
  66. package/dist/records.mjs.map +0 -1
  67. package/dist/representations.d.mts +0 -36
  68. package/dist/representations.d.mts.map +0 -1
  69. package/dist/representations.mjs +0 -44
  70. package/dist/representations.mjs.map +0 -1
  71. package/dist/search-store.d.mts +0 -77
  72. package/dist/search-store.d.mts.map +0 -1
  73. package/dist/search-store.mjs +0 -7
  74. package/dist/search-store.mjs.map +0 -1
  75. package/dist/vector-store.d.mts +0 -3
  76. package/dist/vector-store.mjs +0 -2
  77. package/src/artifacts.ts +0 -48
  78. package/src/links.ts +0 -23
  79. package/src/provenance.ts +0 -25
  80. package/src/records.ts +0 -76
  81. package/src/representations.ts +0 -51
  82. package/src/search-store.ts +0 -98
  83. package/src/vector-store.ts +0 -6
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # @yolk-sdk/knowledge
2
2
 
3
- Domain-free knowledge record, artifact, representation, search, provenance, link, store, and context contracts.
3
+ Simple durable knowledge documents for agents: pinned, searchable, or archived.
4
4
 
5
5
  ## Install
6
6
 
@@ -10,98 +10,77 @@ pnpm add @yolk-sdk/knowledge@canary @yolk-sdk/agent@canary effect
10
10
 
11
11
  Canary APIs are unstable. Keep all `@yolk-sdk/*` packages on the same version.
12
12
 
13
+ ## Model
14
+
15
+ ```txt
16
+ KnowledgeDocument
17
+ id
18
+ slug
19
+ title
20
+ purpose
21
+ origin
22
+ content
23
+ summary?
24
+ status: processing | ready | error
25
+ availability: pinned | searchable | archived
26
+ ```
27
+
28
+ - `pinned`: host may inject into startup context and search.
29
+ - `searchable`: host may expose through search tools.
30
+ - `archived`: retained, normally omitted from prompt and search.
31
+
32
+ Optional files and chunks are modeled as `KnowledgeFile` and `KnowledgeChunk`.
33
+
13
34
  ## Subpaths
14
35
 
15
36
  | Subpath | Purpose |
16
37
  | --- | --- |
17
- | `@yolk-sdk/knowledge` | Root context/store/artifact/error helpers |
18
- | `@yolk-sdk/knowledge/records` | Knowledge record schemas and roles |
19
- | `@yolk-sdk/knowledge/artifacts` | Artifact catalog and blob-store contract |
20
- | `@yolk-sdk/knowledge/representations` | Agent-readable/searchable representations |
21
- | `@yolk-sdk/knowledge/chunking` | Search chunker contracts and defaults |
38
+ | `@yolk-sdk/knowledge` | Root context/store/file/error helpers |
39
+ | `@yolk-sdk/knowledge/documents` | Document, file, chunk, status, availability, and scope schemas |
40
+ | `@yolk-sdk/knowledge/files` | File blob-store contract |
41
+ | `@yolk-sdk/knowledge/store` | Document store and search-index store contracts |
42
+ | `@yolk-sdk/knowledge/context` | Pinned context builder |
43
+ | `@yolk-sdk/knowledge/chunking` | Text chunker contracts and defaults |
22
44
  | `@yolk-sdk/knowledge/embeddings` | Embedder contract and vector types |
23
- | `@yolk-sdk/knowledge/errors` | Shared knowledge error types |
24
- | `@yolk-sdk/knowledge/extraction` | Loaded source/extractor contract |
25
- | `@yolk-sdk/knowledge/ingestion` | Generic search ingestion pipeline |
26
- | `@yolk-sdk/knowledge/search` | Search interface, hybrid search, context packing |
27
- | `@yolk-sdk/knowledge/documents` | Collection, document, chunk, and search scope schemas |
28
- | `@yolk-sdk/knowledge/search-store` | Collection/document/chunk/search lifecycle store |
29
- | `@yolk-sdk/knowledge/vector-store` | Compatibility aliases for vector search/store types |
45
+ | `@yolk-sdk/knowledge/extraction` | Source/extractor contract |
46
+ | `@yolk-sdk/knowledge/ingestion` | Search chunk ingestion helper |
47
+ | `@yolk-sdk/knowledge/search` | Vector/hybrid chunk search helpers |
30
48
  | `@yolk-sdk/knowledge/summarization` | Optional title/summary service contract |
31
- | `@yolk-sdk/knowledge/provenance` | Source/provenance metadata |
32
- | `@yolk-sdk/knowledge/links` | Typed links between records |
33
- | `@yolk-sdk/knowledge/store` | Knowledge store lifecycle contract |
34
- | `@yolk-sdk/knowledge/context` | Context assembly helpers |
35
- | `@yolk-sdk/knowledge/agent` | Agent-facing helper contracts |
49
+ | `@yolk-sdk/knowledge/agent` | `knowledge_lookup` and `knowledge_manage` tool factories |
50
+ | `@yolk-sdk/knowledge/errors` | Shared error types |
36
51
 
37
52
  ## Imports
38
53
 
39
54
  ```ts
40
55
  import { buildKnowledgeContext, KnowledgeStore } from '@yolk-sdk/knowledge'
41
- import type { KnowledgeRecord } from '@yolk-sdk/knowledge/records'
42
- import { KnowledgeArtifactStore } from '@yolk-sdk/knowledge/artifacts'
43
- import { makeDefaultKnowledgeChunker } from '@yolk-sdk/knowledge/chunking'
44
- import { searchKnowledge } from '@yolk-sdk/knowledge/search'
56
+ import type { KnowledgeDocument } from '@yolk-sdk/knowledge/documents'
57
+ import { KnowledgeFileBlobStore } from '@yolk-sdk/knowledge/files'
58
+ import { makeKnowledgeLookupTool, makeKnowledgeManageTool } from '@yolk-sdk/knowledge/agent'
45
59
  ```
46
60
 
47
- ## Migration
48
-
49
- Search APIs use canonical search naming. Removed retrieval APIs are intentionally not backward-compatible.
61
+ ## Use pinned knowledge as context
50
62
 
51
- | Removed | Use |
52
- | --- | --- |
53
- | Retrieval subpath | `@yolk-sdk/knowledge/search` |
54
- | `KnowledgeRetriever` | `KnowledgeSearcher` |
55
- | `retrieve(input)` | `search(input)` |
56
- | `KnowledgeRetrievalError` | `KnowledgeSearchError` |
63
+ ```ts
64
+ import { buildKnowledgeContext } from '@yolk-sdk/knowledge/context'
57
65
 
58
- ## Glossary
66
+ const context = buildKnowledgeContext({
67
+ documents,
68
+ maxCharacters: 6000
69
+ })
70
+ ```
59
71
 
60
- | Term | Meaning |
61
- | --- | --- |
62
- | Knowledge record | Stable logical thing an agent can know about. Domain-free; hosts decide product meaning. |
63
- | Artifact | Stored source/blob attached to knowledge, such as a file or captured output. |
64
- | Representation | Agent-readable or searchable form of knowledge, usually text plus metadata. |
65
- | Provenance | Source metadata explaining where knowledge came from and how it was produced. |
66
- | Link | Typed relationship between knowledge records. |
67
- | Context | Model-ready knowledge selected for a run. Built from host policy. |
68
- | Context policy | Host intent for use: `pinned`, `routable`, `searchable`, or `archived`. |
69
- | Search index | Searchable corpus built from source documents, chunks, embeddings, and metadata. |
70
- | Collection | Named searchable document group with shared embedding and chunking config. |
71
- | Document | Source item tracked through search ingestion: pending, processing, ready, or error. |
72
- | Source | Document origin: file ref, URL, or raw text. |
73
- | Chunk | Searchable slice of a document with position and token count. |
74
- | Embedding | Numeric vector for semantic search. Hosts provide the embedding provider. |
75
- | Chunker | Service that splits extracted text into chunks. |
76
- | Extractor | Service that loads source content into text/title/summary/metadata. |
77
- | Ingestion | Pipeline that extracts, chunks, embeds, stores chunks, and marks document status. |
78
- | Search | Query-time search over search chunks, optionally packed into context text. |
79
- | Hybrid search | Search mode that fuses vector and text search rankings. |
80
- | Search scope | Caller-provided collection or collections to search. Package does not interpret tenant semantics. |
81
- | Search index store | Persistence contract for collections, documents, chunks, status, and search. |
82
- | Vector store | Compatibility naming over search-store chunk search contracts. |
83
- | Knowledge store | Persistence contract for non-index knowledge records and context assembly. |
84
- | Artifact store | Persistence contract for artifact bytes/metadata. |
85
- | Host | App using the package; owns auth, DB, object storage, providers, permissions, and policy. |
86
-
87
- ## Context policy
88
-
89
- | Policy | Meaning |
90
- | --- | --- |
91
- | `pinned` | Host may inject into model startup context |
92
- | `routable` | Host may use for routing/dispatch decisions |
93
- | `searchable` | Host may expose through search tools |
94
- | `archived` | Retained but normally omitted from active context/search |
72
+ Add the resulting text to your system prompt. Your app chooses which documents are pinned.
95
73
 
96
74
  ## Host responsibilities
97
75
 
98
- - Own users, teams, permissions, routing, and product policy.
99
- - Implement `KnowledgeStore` and `KnowledgeArtifactStore` with app storage.
100
- - Own concrete extraction, embeddings, search ingestion, R2/S3 layout, and DB schema.
101
- - Decide which knowledge is pinned, searchable, routable, or archived.
76
+ - Own users, workspaces, permissions, routing, and product policy.
77
+ - Implement `KnowledgeStore` with app storage.
78
+ - Implement `KnowledgeFileBlobStore` with R2/S3/blob storage when files are needed.
79
+ - Own concrete extraction, embeddings, search SQL/vector storage, and upload URLs.
80
+ - Keep slug uniqueness scoped to the host boundary.
102
81
 
103
82
  ## Boundaries
104
83
 
105
84
  - No app auth, DB drivers, object storage SDKs, React, Next.js, or provider SDKs.
106
- - Search/chunking APIs are exposed as flat knowledge subpaths; concrete stores and providers remain app-owned.
107
- - Package owns semantics and contracts only.
85
+ - Package owns simple schemas, contracts, context formatting, tool factories, chunking, and search helpers.
86
+ - Host owns all IO and policy.
package/dist/agent.d.mts CHANGED
@@ -1,26 +1,72 @@
1
- import { KnowledgeScope } from "./records.mjs";
2
- import { KnowledgeSearchScope } from "./documents.mjs";
3
- import { KnowledgeSearcher } from "./search.mjs";
1
+ import { KnowledgeAvailability, KnowledgeDocument } from "./documents.mjs";
4
2
  import { Effect } from "effect";
5
3
  import { ToolError } from "@yolk-sdk/agent/loop";
6
4
  import { ToolRegistration } from "@yolk-sdk/agent/tools";
7
5
 
8
6
  //#region src/agent.d.ts
9
- type ResolveKnowledgeScope<Context> = (context: Context) => KnowledgeScope;
10
- type KnowledgeAgentContextOptions = {
11
- readonly maxPinnedContextCharacters: number;
7
+ type KnowledgeTarget = {
8
+ readonly slug: string;
9
+ readonly scope?: string;
12
10
  };
13
- declare const defaultKnowledgeAgentContextOptions: KnowledgeAgentContextOptions;
14
- type KnowledgeSearchToolScopeResolver<Context> = KnowledgeSearchScope | ((context: Context) => Effect.Effect<KnowledgeSearchScope, ToolError>);
15
- type MakeKnowledgeSearchToolOptions<Context> = {
16
- readonly scope: KnowledgeSearchToolScopeResolver<Context>;
11
+ type KnowledgeLookupResult = {
12
+ readonly document: KnowledgeDocument;
13
+ readonly score?: number;
14
+ readonly context?: ReadonlyArray<{
15
+ readonly content: string;
16
+ }>;
17
+ };
18
+ type KnowledgeSavedDocument = {
19
+ readonly id: string;
20
+ readonly slug: string;
21
+ readonly title: string;
22
+ };
23
+ type KnowledgeLookupHandlers<Context> = {
24
+ readonly search: (input: {
25
+ readonly context: Context;
26
+ readonly query: string;
27
+ readonly limit?: number;
28
+ readonly minScore?: number;
29
+ readonly contextChunks?: number;
30
+ }) => Effect.Effect<ReadonlyArray<KnowledgeLookupResult>, ToolError>;
31
+ readonly get: (input: {
32
+ readonly context: Context;
33
+ readonly id?: string;
34
+ readonly target?: KnowledgeTarget;
35
+ }) => Effect.Effect<KnowledgeDocument, ToolError>;
36
+ };
37
+ type KnowledgeManageHandlers<Context> = {
38
+ readonly upsert: (input: {
39
+ readonly context: Context;
40
+ readonly target: KnowledgeTarget;
41
+ readonly title: string;
42
+ readonly purpose: string;
43
+ readonly origin: string;
44
+ readonly content: string;
45
+ readonly availability: KnowledgeAvailability;
46
+ }) => Effect.Effect<KnowledgeSavedDocument, ToolError>;
47
+ readonly setAvailability: (input: {
48
+ readonly context: Context;
49
+ readonly target: KnowledgeTarget;
50
+ readonly availability: KnowledgeAvailability;
51
+ }) => Effect.Effect<KnowledgeSavedDocument, ToolError>;
52
+ readonly renameSlug: (input: {
53
+ readonly context: Context;
54
+ readonly target: KnowledgeTarget;
55
+ readonly nextSlug: string;
56
+ }) => Effect.Effect<KnowledgeSavedDocument, ToolError>;
57
+ readonly delete: (input: {
58
+ readonly context: Context;
59
+ readonly target: KnowledgeTarget;
60
+ }) => Effect.Effect<KnowledgeSavedDocument, ToolError>;
61
+ };
62
+ declare const makeKnowledgeLookupTool: <Context>(handlers: KnowledgeLookupHandlers<Context>, options?: {
17
63
  readonly name?: string;
18
64
  readonly description?: string;
19
- readonly limit?: number;
20
- readonly minScore?: number;
21
- readonly contextChunks?: number;
22
- };
23
- declare const makeKnowledgeSearchTool: <Context>(searcher: KnowledgeSearcher, options: MakeKnowledgeSearchToolOptions<Context>) => ToolRegistration<Context>;
65
+ }) => ToolRegistration<Context>;
66
+ declare const makeKnowledgeManageTool: <Context>(handlers: KnowledgeManageHandlers<Context>, options?: {
67
+ readonly name?: string;
68
+ readonly description?: string;
69
+ }) => ToolRegistration<Context>;
24
70
  //#endregion
25
- export { KnowledgeAgentContextOptions, KnowledgeSearchToolScopeResolver, MakeKnowledgeSearchToolOptions, ResolveKnowledgeScope, defaultKnowledgeAgentContextOptions, makeKnowledgeSearchTool };
71
+ export { KnowledgeLookupHandlers, KnowledgeLookupResult, KnowledgeManageHandlers, KnowledgeSavedDocument, KnowledgeTarget, makeKnowledgeLookupTool, makeKnowledgeManageTool };
26
72
  //# sourceMappingURL=agent.d.mts.map
@@ -1 +1 @@
1
- {"version":3,"file":"agent.d.mts","names":[],"sources":["../src/agent.ts"],"mappings":";;;;;;;;KAUY,qBAAA,aAAkC,OAAA,EAAS,OAAA,KAAY,cAAc;AAAA,KAErE,4BAAA;EAAA,SACD,0BAA0B;AAAA;AAAA,cAGxB,mCAAA,EAAqC,4BAEjD;AAAA,KAWW,gCAAA,YACR,oBAAA,KACE,OAAA,EAAS,OAAA,KAAY,MAAA,CAAO,MAAA,CAAO,oBAAA,EAAsB,SAAA;AAAA,KAEnD,8BAAA;EAAA,SACD,KAAA,EAAO,gCAAgC,CAAC,OAAA;EAAA,SACxC,IAAA;EAAA,SACA,WAAA;EAAA,SACA,KAAA;EAAA,SACA,QAAA;EAAA,SACA,aAAA;AAAA;AAAA,cAWE,uBAAA,YACX,QAAA,EAAU,iBAAA,EACV,OAAA,EAAS,8BAAA,CAA+B,OAAA,MACvC,gBAAA,CAAiB,OAAA"}
1
+ {"version":3,"file":"agent.d.mts","names":[],"sources":["../src/agent.ts"],"mappings":";;;;;;KAiEY,eAAA;EAAA,SACD,IAAA;EAAA,SACA,KAAK;AAAA;AAAA,KAGJ,qBAAA;EAAA,SACD,QAAA,EAAU,iBAAA;EAAA,SACV,KAAA;EAAA,SACA,OAAA,GAAU,aAAa;IAAA,SAAY,OAAA;EAAA;AAAA;AAAA,KAGlC,sBAAA;EAAA,SACD,EAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA;AAAA;AAAA,KAGC,uBAAA;EAAA,SACD,MAAA,GAAS,KAAA;IAAA,SACP,OAAA,EAAS,OAAA;IAAA,SACT,KAAA;IAAA,SACA,KAAA;IAAA,SACA,QAAA;IAAA,SACA,aAAA;EAAA,MACL,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,qBAAA,GAAwB,SAAA;EAAA,SACjD,GAAA,GAAM,KAAA;IAAA,SACJ,OAAA,EAAS,OAAA;IAAA,SACT,EAAA;IAAA,SACA,MAAA,GAAS,eAAA;EAAA,MACd,MAAA,CAAO,MAAA,CAAO,iBAAA,EAAmB,SAAA;AAAA;AAAA,KAG7B,uBAAA;EAAA,SACD,MAAA,GAAS,KAAA;IAAA,SACP,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;IAAA,SACR,KAAA;IAAA,SACA,OAAA;IAAA,SACA,MAAA;IAAA,SACA,OAAA;IAAA,SACA,YAAA,EAAc,qBAAA;EAAA,MACnB,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;EAAA,SACnC,eAAA,GAAkB,KAAA;IAAA,SAChB,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;IAAA,SACR,YAAA,EAAc,qBAAA;EAAA,MACnB,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;EAAA,SACnC,UAAA,GAAa,KAAA;IAAA,SACX,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;IAAA,SACR,QAAA;EAAA,MACL,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;EAAA,SACnC,MAAA,GAAS,KAAA;IAAA,SACP,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;EAAA,MACb,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;AAAA;AAAA,cAoDjC,uBAAA,YACX,QAAA,EAAU,uBAAA,CAAwB,OAAA,GAClC,OAAA;EAAA,SAAoB,IAAA;EAAA,SAAwB,WAAA;AAAA,MAC3C,gBAAA,CAAiB,OAAA;AAAA,cAmCP,uBAAA,YACX,QAAA,EAAU,uBAAA,CAAwB,OAAA,GAClC,OAAA;EAAA,SAAoB,IAAA;EAAA,SAAwB,WAAA;AAAA,MAC3C,gBAAA,CAAiB,OAAA"}
package/dist/agent.mjs CHANGED
@@ -1,45 +1,186 @@
1
- import { packKnowledgeSearchContext } from "./search.mjs";
1
+ import { KnowledgeAvailabilitySchema, NonEmptyTrimmedString } from "./documents.mjs";
2
2
  import { Effect } from "effect";
3
- import * as Schema$1 from "effect/Schema";
3
+ import * as Schema from "effect/Schema";
4
4
  import { ToolError } from "@yolk-sdk/agent/loop";
5
5
  import { ToolResult } from "@yolk-sdk/agent/protocol";
6
6
  import { makeTool } from "@yolk-sdk/agent/tools";
7
7
  //#region src/agent.ts
8
- const defaultKnowledgeAgentContextOptions = { maxPinnedContextCharacters: 6e3 };
9
- const KnowledgeSearchToolParams = Schema$1.Struct({ query: Schema$1.Trimmed.pipe(Schema$1.check(Schema$1.isNonEmpty()), Schema$1.annotate({ description: "Search query for the configured knowledge search." })) });
10
- const isToolError = Schema$1.is(ToolError);
11
- const resolveScope = (resolver, context) => {
12
- if (typeof resolver === "function") return resolver(context);
13
- return Effect.succeed(resolver);
8
+ const OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString));
9
+ const OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number));
10
+ const KnowledgeTargetParamsSchema = Schema.Struct({
11
+ slug: NonEmptyTrimmedString,
12
+ scope: OptionalNonEmptyTrimmedString
13
+ });
14
+ const KnowledgeLookupParams = Schema.Union([Schema.Struct({
15
+ operation: Schema.Literal("search"),
16
+ query: NonEmptyTrimmedString,
17
+ limit: OptionalNumber,
18
+ minScore: OptionalNumber,
19
+ contextChunks: OptionalNumber
20
+ }), Schema.Struct({
21
+ operation: Schema.Literal("get"),
22
+ id: OptionalNonEmptyTrimmedString,
23
+ target: Schema.optional(Schema.NullOr(KnowledgeTargetParamsSchema))
24
+ })]);
25
+ const KnowledgeManageParams = Schema.Union([
26
+ Schema.Struct({
27
+ operation: Schema.Literal("upsert"),
28
+ target: KnowledgeTargetParamsSchema,
29
+ title: NonEmptyTrimmedString,
30
+ purpose: NonEmptyTrimmedString,
31
+ origin: NonEmptyTrimmedString,
32
+ content: NonEmptyTrimmedString,
33
+ availability: KnowledgeAvailabilitySchema
34
+ }),
35
+ Schema.Struct({
36
+ operation: Schema.Literal("set_availability"),
37
+ target: KnowledgeTargetParamsSchema,
38
+ availability: KnowledgeAvailabilitySchema
39
+ }),
40
+ Schema.Struct({
41
+ operation: Schema.Literal("rename_slug"),
42
+ target: KnowledgeTargetParamsSchema,
43
+ nextSlug: NonEmptyTrimmedString
44
+ }),
45
+ Schema.Struct({
46
+ operation: Schema.Literal("delete"),
47
+ target: KnowledgeTargetParamsSchema
48
+ })
49
+ ]);
50
+ const unknownToMessage = (error) => error instanceof Error ? error.message : String(error);
51
+ const optionalValue = (value) => value ?? void 0;
52
+ const targetFromParams = (target) => target.scope === null || target.scope === void 0 ? { slug: target.slug } : {
53
+ slug: target.slug,
54
+ scope: target.scope
14
55
  };
15
- const makeKnowledgeSearchTool = (searcher, options) => {
56
+ const optionalTargetFromParams = (target) => target === null || target === void 0 ? void 0 : targetFromParams(target);
57
+ const formatDocument = (document) => [
58
+ `# ${document.title}`,
59
+ `document_id: ${document.id}`,
60
+ `slug: ${document.slug}`,
61
+ `purpose: ${document.purpose}`,
62
+ `origin: ${document.origin}`,
63
+ `availability: ${document.availability}`,
64
+ `status: ${document.status}`,
65
+ "",
66
+ document.content
67
+ ].join("\n");
68
+ const formatSearchResults = (results) => results.length === 0 ? "No knowledge matches found." : results.map((result) => [
69
+ `## ${result.document.title}`,
70
+ `document_id: ${result.document.id}`,
71
+ `slug: ${result.document.slug}`,
72
+ `purpose: ${result.document.purpose}`,
73
+ `origin: ${result.document.origin}`,
74
+ `availability: ${result.document.availability}`,
75
+ result.score === void 0 ? void 0 : `score: ${result.score}`,
76
+ "",
77
+ result.context?.map((chunk) => chunk.content).join("\n\n") ?? result.document.content
78
+ ].filter((line) => line !== void 0).join("\n")).join("\n\n");
79
+ const formatSaved = (verb, document) => [
80
+ verb,
81
+ `title: ${document.title}`,
82
+ `slug: ${document.slug}`,
83
+ `document_id: ${document.id}`
84
+ ].join("\n");
85
+ const makeKnowledgeLookupTool = (handlers, options = {}) => {
16
86
  return makeTool({
17
- name: options.name ?? "search_knowledge",
18
- description: options.description ?? "Search the configured knowledge search.",
19
- parameters: KnowledgeSearchToolParams,
87
+ name: options.name ?? "knowledge_lookup",
88
+ description: options.description ?? "Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.",
89
+ parameters: KnowledgeLookupParams,
20
90
  access: "read",
21
- invalidParamsMessage: (error) => error instanceof Error ? error.message : String(error),
22
- execute: (input) => resolveScope(options.scope, input.context).pipe(Effect.flatMap((scope) => searcher.search({
23
- scope,
24
- query: input.params.query,
25
- limit: options.limit,
26
- minScore: options.minScore,
27
- contextChunks: options.contextChunks
28
- })), Effect.map((results) => packKnowledgeSearchContext(input.params.query, results)), Effect.map((context) => ToolResult.make({
29
- toolCallId: input.call.id,
30
- content: context.text,
31
- structuredContent: context
32
- })), Effect.mapError((error) => {
33
- if (isToolError(error)) return error;
34
- return new ToolError({
35
- tool: input.call.name,
36
- message: error.message,
37
- cause: "execution"
91
+ invalidParamsMessage: (error) => `Invalid knowledge lookup arguments: ${unknownToMessage(error)}`,
92
+ execute: ({ call, context, params }) => Effect.gen(function* () {
93
+ if (params.operation === "search") {
94
+ const results = yield* handlers.search({
95
+ context,
96
+ query: params.query,
97
+ limit: optionalValue(params.limit),
98
+ minScore: optionalValue(params.minScore),
99
+ contextChunks: optionalValue(params.contextChunks)
100
+ });
101
+ return ToolResult.make({
102
+ toolCallId: call.id,
103
+ content: formatSearchResults(results)
104
+ });
105
+ }
106
+ const document = yield* handlers.get({
107
+ context,
108
+ id: optionalValue(params.id),
109
+ target: optionalTargetFromParams(params.target)
38
110
  });
39
- }))
111
+ return ToolResult.make({
112
+ toolCallId: call.id,
113
+ content: formatDocument(document)
114
+ });
115
+ })
116
+ });
117
+ };
118
+ const makeKnowledgeManageTool = (handlers, options = {}) => {
119
+ return makeTool({
120
+ name: options.name ?? "knowledge_manage",
121
+ description: options.description ?? "Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.",
122
+ parameters: KnowledgeManageParams,
123
+ access: "write",
124
+ invalidParamsMessage: (error) => `Invalid knowledge manage arguments: ${unknownToMessage(error)}`,
125
+ execute: ({ call, context, params }) => Effect.gen(function* () {
126
+ switch (params.operation) {
127
+ case "upsert": {
128
+ const document = yield* handlers.upsert({
129
+ context,
130
+ target: targetFromParams(params.target),
131
+ title: params.title,
132
+ purpose: params.purpose,
133
+ origin: params.origin,
134
+ content: params.content,
135
+ availability: params.availability
136
+ });
137
+ return ToolResult.make({
138
+ toolCallId: call.id,
139
+ content: formatSaved("Knowledge upserted", document)
140
+ });
141
+ }
142
+ case "set_availability": {
143
+ const document = yield* handlers.setAvailability({
144
+ context,
145
+ target: targetFromParams(params.target),
146
+ availability: params.availability
147
+ });
148
+ return ToolResult.make({
149
+ toolCallId: call.id,
150
+ content: formatSaved("Knowledge availability updated", document)
151
+ });
152
+ }
153
+ case "rename_slug": {
154
+ const document = yield* handlers.renameSlug({
155
+ context,
156
+ target: targetFromParams(params.target),
157
+ nextSlug: params.nextSlug
158
+ });
159
+ return ToolResult.make({
160
+ toolCallId: call.id,
161
+ content: formatSaved("Knowledge slug renamed", document)
162
+ });
163
+ }
164
+ case "delete": {
165
+ const document = yield* handlers.delete({
166
+ context,
167
+ target: targetFromParams(params.target)
168
+ });
169
+ return ToolResult.make({
170
+ toolCallId: call.id,
171
+ content: formatSaved("Knowledge deleted", document)
172
+ });
173
+ }
174
+ }
175
+ return yield* Effect.fail(new ToolError({
176
+ tool: call.name,
177
+ message: "Unsupported knowledge operation",
178
+ cause: "validation"
179
+ }));
180
+ })
40
181
  });
41
182
  };
42
183
  //#endregion
43
- export { defaultKnowledgeAgentContextOptions, makeKnowledgeSearchTool };
184
+ export { makeKnowledgeLookupTool, makeKnowledgeManageTool };
44
185
 
45
186
  //# sourceMappingURL=agent.mjs.map
@@ -1 +1 @@
1
- {"version":3,"file":"agent.mjs","names":["Schema"],"sources":["../src/agent.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport * as Schema from 'effect/Schema'\nimport { ToolError } from '@yolk-sdk/agent/loop'\nimport { ToolResult } from '@yolk-sdk/agent/protocol'\nimport { makeTool, type ToolRegistration } from '@yolk-sdk/agent/tools'\nimport type { KnowledgeScope } from './records.ts'\nimport type { KnowledgeSearchScope } from './documents.ts'\nimport type { KnowledgeSearcher } from './search.ts'\nimport { packKnowledgeSearchContext } from './search.ts'\n\nexport type ResolveKnowledgeScope<Context> = (context: Context) => KnowledgeScope\n\nexport type KnowledgeAgentContextOptions = {\n readonly maxPinnedContextCharacters: number\n}\n\nexport const defaultKnowledgeAgentContextOptions: KnowledgeAgentContextOptions = {\n maxPinnedContextCharacters: 6000\n}\n\nconst KnowledgeSearchToolParams = Schema.Struct({\n query: Schema.Trimmed.pipe(\n Schema.check(Schema.isNonEmpty()),\n Schema.annotate({ description: 'Search query for the configured knowledge search.' })\n )\n})\n\nconst isToolError = Schema.is(ToolError)\n\nexport type KnowledgeSearchToolScopeResolver<Context> =\n | KnowledgeSearchScope\n | ((context: Context) => Effect.Effect<KnowledgeSearchScope, ToolError>)\n\nexport type MakeKnowledgeSearchToolOptions<Context> = {\n readonly scope: KnowledgeSearchToolScopeResolver<Context>\n readonly name?: string\n readonly description?: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n}\n\nconst resolveScope = <Context>(resolver: KnowledgeSearchToolScopeResolver<Context>, context: Context) => {\n if (typeof resolver === 'function') {\n return resolver(context)\n }\n\n return Effect.succeed(resolver)\n}\n\nexport const makeKnowledgeSearchTool = <Context>(\n searcher: KnowledgeSearcher,\n options: MakeKnowledgeSearchToolOptions<Context>\n): ToolRegistration<Context> => {\n const name = options.name ?? 'search_knowledge'\n\n return makeTool({\n name,\n description: options.description ?? 'Search the configured knowledge search.',\n parameters: KnowledgeSearchToolParams,\n access: 'read',\n invalidParamsMessage: error => error instanceof Error ? error.message : String(error),\n execute: input =>\n resolveScope(options.scope, input.context).pipe(\n Effect.flatMap(scope =>\n searcher.search({\n scope,\n query: input.params.query,\n limit: options.limit,\n minScore: options.minScore,\n contextChunks: options.contextChunks\n })\n ),\n Effect.map(results => packKnowledgeSearchContext(input.params.query, results)),\n Effect.map(\n context =>\n ToolResult.make({\n toolCallId: input.call.id,\n content: context.text,\n structuredContent: context\n })\n ),\n Effect.mapError(error => {\n if (isToolError(error)) {\n return error\n }\n\n return new ToolError({\n tool: input.call.name,\n message: error.message,\n cause: 'execution'\n })\n })\n )\n })\n}\n"],"mappings":";;;;;;;AAgBA,MAAa,sCAAoE,EAC/E,4BAA4B,IAC9B;AAEA,MAAM,4BAA4BA,SAAO,OAAO,EAC9C,OAAOA,SAAO,QAAQ,KACpBA,SAAO,MAAMA,SAAO,WAAW,CAAC,GAChCA,SAAO,SAAS,EAAE,aAAa,oDAAoD,CAAC,CACtF,EACF,CAAC;AAED,MAAM,cAAcA,SAAO,GAAG,SAAS;AAevC,MAAM,gBAAyB,UAAqD,YAAqB;CACvG,IAAI,OAAO,aAAa,YACtB,OAAO,SAAS,OAAO;CAGzB,OAAO,OAAO,QAAQ,QAAQ;AAChC;AAEA,MAAa,2BACX,UACA,YAC8B;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aAAa,QAAQ,eAAe;EACpC,YAAY;EACZ,QAAQ;EACR,uBAAsB,UAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACpF,UAAS,UACP,aAAa,QAAQ,OAAO,MAAM,OAAO,EAAE,KACzC,OAAO,SAAQ,UACb,SAAS,OAAO;GACd;GACA,OAAO,MAAM,OAAO;GACpB,OAAO,QAAQ;GACf,UAAU,QAAQ;GAClB,eAAe,QAAQ;EACzB,CAAC,CACH,GACA,OAAO,KAAI,YAAW,2BAA2B,MAAM,OAAO,OAAO,OAAO,CAAC,GAC7E,OAAO,KACL,YACE,WAAW,KAAK;GACd,YAAY,MAAM,KAAK;GACvB,SAAS,QAAQ;GACjB,mBAAmB;EACrB,CAAC,CACL,GACA,OAAO,UAAS,UAAS;GACvB,IAAI,YAAY,KAAK,GACnB,OAAO;GAGT,OAAO,IAAI,UAAU;IACnB,MAAM,MAAM,KAAK;IACjB,SAAS,MAAM;IACf,OAAO;GACT,CAAC;EACH,CAAC,CACH;CACJ,CAAC;AACH"}
1
+ {"version":3,"file":"agent.mjs","names":[],"sources":["../src/agent.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport * as Schema from 'effect/Schema'\nimport { ToolError } from '@yolk-sdk/agent/loop'\nimport { ToolResult } from '@yolk-sdk/agent/protocol'\nimport { makeTool, type ToolRegistration } from '@yolk-sdk/agent/tools'\nimport {\n KnowledgeAvailabilitySchema,\n NonEmptyTrimmedString,\n type KnowledgeAvailability,\n type KnowledgeDocument\n} from './documents.ts'\n\nconst OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString))\nconst OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number))\n\nconst KnowledgeTargetParamsSchema = Schema.Struct({\n slug: NonEmptyTrimmedString,\n scope: OptionalNonEmptyTrimmedString\n})\n\nconst KnowledgeLookupParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n query: NonEmptyTrimmedString,\n limit: OptionalNumber,\n minScore: OptionalNumber,\n contextChunks: OptionalNumber\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n id: OptionalNonEmptyTrimmedString,\n target: Schema.optional(Schema.NullOr(KnowledgeTargetParamsSchema))\n })\n])\n\nconst KnowledgeManageParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('upsert'),\n target: KnowledgeTargetParamsSchema,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('set_availability'),\n target: KnowledgeTargetParamsSchema,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('rename_slug'),\n target: KnowledgeTargetParamsSchema,\n nextSlug: NonEmptyTrimmedString\n }),\n Schema.Struct({\n operation: Schema.Literal('delete'),\n target: KnowledgeTargetParamsSchema\n })\n])\n\ntype KnowledgeTargetParams = typeof KnowledgeTargetParamsSchema.Type\ntype KnowledgeLookupParams = typeof KnowledgeLookupParams.Type\ntype KnowledgeManageParams = typeof KnowledgeManageParams.Type\n\nexport type KnowledgeTarget = {\n readonly slug: string\n readonly scope?: string\n}\n\nexport type KnowledgeLookupResult = {\n readonly document: KnowledgeDocument\n readonly score?: number\n readonly context?: ReadonlyArray<{ readonly content: string }>\n}\n\nexport type KnowledgeSavedDocument = {\n readonly id: string\n readonly slug: string\n readonly title: string\n}\n\nexport type KnowledgeLookupHandlers<Context> = {\n readonly search: (input: {\n readonly context: Context\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n }) => Effect.Effect<ReadonlyArray<KnowledgeLookupResult>, ToolError>\n readonly get: (input: {\n readonly context: Context\n readonly id?: string\n readonly target?: KnowledgeTarget\n }) => Effect.Effect<KnowledgeDocument, ToolError>\n}\n\nexport type KnowledgeManageHandlers<Context> = {\n readonly upsert: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly title: string\n readonly purpose: string\n readonly origin: string\n readonly content: string\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly setAvailability: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly renameSlug: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly nextSlug: string\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly delete: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n}\n\nconst unknownToMessage = (error: unknown) => error instanceof Error ? error.message : String(error)\n\nconst optionalValue = <Value>(value: Value | null | undefined) => value ?? undefined\n\nconst targetFromParams = (target: KnowledgeTargetParams): KnowledgeTarget =>\n target.scope === null || target.scope === undefined\n ? { slug: target.slug }\n : { slug: target.slug, scope: target.scope }\n\nconst optionalTargetFromParams = (target: KnowledgeTargetParams | null | undefined) =>\n target === null || target === undefined ? undefined : targetFromParams(target)\n\nconst formatDocument = (document: KnowledgeDocument) =>\n [\n `# ${document.title}`,\n `document_id: ${document.id}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `availability: ${document.availability}`,\n `status: ${document.status}`,\n '',\n document.content\n ].join('\\n')\n\nconst formatSearchResults = (results: ReadonlyArray<KnowledgeLookupResult>) =>\n results.length === 0\n ? 'No knowledge matches found.'\n : results\n .map(result =>\n [\n `## ${result.document.title}`,\n `document_id: ${result.document.id}`,\n `slug: ${result.document.slug}`,\n `purpose: ${result.document.purpose}`,\n `origin: ${result.document.origin}`,\n `availability: ${result.document.availability}`,\n result.score === undefined ? undefined : `score: ${result.score}`,\n '',\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.document.content\n ]\n .filter(line => line !== undefined)\n .join('\\n')\n )\n .join('\\n\\n')\n\nconst formatSaved = (verb: string, document: KnowledgeSavedDocument) =>\n [verb, `title: ${document.title}`, `slug: ${document.slug}`, `document_id: ${document.id}`].join('\\n')\n\nexport const makeKnowledgeLookupTool = <Context>(\n handlers: KnowledgeLookupHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_lookup'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.',\n parameters: KnowledgeLookupParams,\n access: 'read',\n invalidParamsMessage: error => `Invalid knowledge lookup arguments: ${unknownToMessage(error)}`,\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n if (params.operation === 'search') {\n const results = yield* handlers.search({\n context,\n query: params.query,\n limit: optionalValue(params.limit),\n minScore: optionalValue(params.minScore),\n contextChunks: optionalValue(params.contextChunks)\n })\n\n return ToolResult.make({ toolCallId: call.id, content: formatSearchResults(results) })\n }\n\n const document = yield* handlers.get({\n context,\n id: optionalValue(params.id),\n target: optionalTargetFromParams(params.target)\n })\n return ToolResult.make({ toolCallId: call.id, content: formatDocument(document) })\n })\n })\n}\n\nexport const makeKnowledgeManageTool = <Context>(\n handlers: KnowledgeManageHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_manage'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.',\n parameters: KnowledgeManageParams,\n access: 'write',\n invalidParamsMessage: error => `Invalid knowledge manage arguments: ${unknownToMessage(error)}`,\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n switch (params.operation) {\n case 'upsert': {\n const document = yield* handlers.upsert({\n context,\n target: targetFromParams(params.target),\n title: params.title,\n purpose: params.purpose,\n origin: params.origin,\n content: params.content,\n availability: params.availability\n })\n return ToolResult.make({ toolCallId: call.id, content: formatSaved('Knowledge upserted', document) })\n }\n case 'set_availability': {\n const document = yield* handlers.setAvailability({\n context,\n target: targetFromParams(params.target),\n availability: params.availability\n })\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge availability updated', document)\n })\n }\n case 'rename_slug': {\n const document = yield* handlers.renameSlug({\n context,\n target: targetFromParams(params.target),\n nextSlug: params.nextSlug\n })\n return ToolResult.make({ toolCallId: call.id, content: formatSaved('Knowledge slug renamed', document) })\n }\n case 'delete': {\n const document = yield* handlers.delete({ context, target: targetFromParams(params.target) })\n return ToolResult.make({ toolCallId: call.id, content: formatSaved('Knowledge deleted', document) })\n }\n }\n\n return yield* Effect.fail(\n new ToolError({ tool: call.name, message: 'Unsupported knowledge operation', cause: 'validation' })\n )\n })\n })\n}\n"],"mappings":";;;;;;;AAYA,MAAM,gCAAgC,OAAO,SAAS,OAAO,OAAO,qBAAqB,CAAC;AAC1F,MAAM,iBAAiB,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC;AAEnE,MAAM,8BAA8B,OAAO,OAAO;CAChD,MAAM;CACN,OAAO;AACT,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM,CACzC,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,OAAO;CACP,OAAO;CACP,UAAU;CACV,eAAe;AACjB,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,IAAI;CACJ,QAAQ,OAAO,SAAS,OAAO,OAAO,2BAA2B,CAAC;AACpE,CAAC,CACH,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM;CACzC,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;EACR,OAAO;EACP,SAAS;EACT,QAAQ;EACR,SAAS;EACT,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,kBAAkB;EAC5C,QAAQ;EACR,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,aAAa;EACvC,QAAQ;EACR,UAAU;CACZ,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;CACV,CAAC;AACH,CAAC;AAgED,MAAM,oBAAoB,UAAmB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAElG,MAAM,iBAAwB,UAAoC,SAAS,KAAA;AAE3E,MAAM,oBAAoB,WACxB,OAAO,UAAU,QAAQ,OAAO,UAAU,KAAA,IACtC,EAAE,MAAM,OAAO,KAAK,IACpB;CAAE,MAAM,OAAO;CAAM,OAAO,OAAO;AAAM;AAE/C,MAAM,4BAA4B,WAChC,WAAW,QAAQ,WAAW,KAAA,IAAY,KAAA,IAAY,iBAAiB,MAAM;AAE/E,MAAM,kBAAkB,aACtB;CACE,KAAK,SAAS;CACd,gBAAgB,SAAS;CACzB,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,WAAW,SAAS;CACpB;CACA,SAAS;AACX,EAAE,KAAK,IAAI;AAEb,MAAM,uBAAuB,YAC3B,QAAQ,WAAW,IACf,gCACA,QACG,KAAI,WACH;CACE,MAAM,OAAO,SAAS;CACtB,gBAAgB,OAAO,SAAS;CAChC,SAAS,OAAO,SAAS;CACzB,YAAY,OAAO,SAAS;CAC5B,WAAW,OAAO,SAAS;CAC3B,iBAAiB,OAAO,SAAS;CACjC,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,UAAU,OAAO;CAC1D;CACA,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,SAAS;AAC9E,EACG,QAAO,SAAQ,SAAS,KAAA,CAAS,EACjC,KAAK,IAAI,CACd,EACC,KAAK,MAAM;AAEpB,MAAM,eAAe,MAAc,aACjC;CAAC;CAAM,UAAU,SAAS;CAAS,SAAS,SAAS;CAAQ,gBAAgB,SAAS;AAAI,EAAE,KAAK,IAAI;AAEvG,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,uBAAsB,UAAS,uCAAuC,iBAAiB,KAAK;EAC5F,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,IAAI,OAAO,cAAc,UAAU;IACjC,MAAM,UAAU,OAAO,SAAS,OAAO;KACrC;KACA,OAAO,OAAO;KACd,OAAO,cAAc,OAAO,KAAK;KACjC,UAAU,cAAc,OAAO,QAAQ;KACvC,eAAe,cAAc,OAAO,aAAa;IACnD,CAAC;IAED,OAAO,WAAW,KAAK;KAAE,YAAY,KAAK;KAAI,SAAS,oBAAoB,OAAO;IAAE,CAAC;GACvF;GAEA,MAAM,WAAW,OAAO,SAAS,IAAI;IACnC;IACA,IAAI,cAAc,OAAO,EAAE;IAC3B,QAAQ,yBAAyB,OAAO,MAAM;GAChD,CAAC;GACD,OAAO,WAAW,KAAK;IAAE,YAAY,KAAK;IAAI,SAAS,eAAe,QAAQ;GAAE,CAAC;EACnF,CAAC;CACL,CAAC;AACH;AAEA,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,uBAAsB,UAAS,uCAAuC,iBAAiB,KAAK;EAC5F,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,QAAQ,OAAO,WAAf;IACE,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,OAAO,OAAO;MACd,SAAS,OAAO;MAChB,QAAQ,OAAO;MACf,SAAS,OAAO;MAChB,cAAc,OAAO;KACvB,CAAC;KACD,OAAO,WAAW,KAAK;MAAE,YAAY,KAAK;MAAI,SAAS,YAAY,sBAAsB,QAAQ;KAAE,CAAC;IACtG;IACA,KAAK,oBAAoB;KACvB,MAAM,WAAW,OAAO,SAAS,gBAAgB;MAC/C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,cAAc,OAAO;KACvB,CAAC;KACD,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,kCAAkC,QAAQ;KACjE,CAAC;IACH;IACA,KAAK,eAAe;KAClB,MAAM,WAAW,OAAO,SAAS,WAAW;MAC1C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,UAAU,OAAO;KACnB,CAAC;KACD,OAAO,WAAW,KAAK;MAAE,YAAY,KAAK;MAAI,SAAS,YAAY,0BAA0B,QAAQ;KAAE,CAAC;IAC1G;IACA,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MAAE;MAAS,QAAQ,iBAAiB,OAAO,MAAM;KAAE,CAAC;KAC5F,OAAO,WAAW,KAAK;MAAE,YAAY,KAAK;MAAI,SAAS,YAAY,qBAAqB,QAAQ;KAAE,CAAC;IACrG;GACF;GAEA,OAAO,OAAO,OAAO,KACnB,IAAI,UAAU;IAAE,MAAM,KAAK;IAAM,SAAS;IAAmC,OAAO;GAAa,CAAC,CACpG;EACF,CAAC;CACL,CAAC;AACH"}
@@ -4,7 +4,7 @@ import { Context, Effect, Layer } from "effect";
4
4
 
5
5
  //#region src/chunking.d.ts
6
6
  type ChunkKnowledgeDocumentInput = {
7
- readonly collectionId: string;
7
+ readonly scopeId: string;
8
8
  readonly documentId: string;
9
9
  readonly content: string;
10
10
  readonly maxTokens?: number;
@@ -18,7 +18,7 @@ declare class KnowledgeChunker extends KnowledgeChunker_base {}
18
18
  declare const countKnowledgeChunkTokens: (text: string) => number;
19
19
  declare const chunkKnowledgeText: (input: ChunkKnowledgeDocumentInput, maxTokens: number) => Effect.Effect<{
20
20
  id: string;
21
- collectionId: string;
21
+ scopeId: string;
22
22
  documentId: string;
23
23
  content: string;
24
24
  position: number;
@@ -1 +1 @@
1
- {"version":3,"file":"chunking.d.mts","names":[],"sources":["../src/chunking.ts"],"mappings":";;;;;KASY,2BAAA;EAAA,SACD,YAAA;EAAA,SACA,UAAA;EAAA,SACA,OAAA;EAAA,SACA,SAAA;EAAA,SACA,QAAA,GAAW,iBAAiB;AAAA;AAAA,KAG3B,mBAAA;EAAA,SACD,KAAA,GACP,KAAA,EAAO,2BAAA,KACJ,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,cAAA,GAAiB,sBAAA;AAAA;AAAA,cACnD,qBAAA;cAEY,gBAAA,SAAyB,qBAErC;AAAA,cAEY,yBAAA,GAA6B,IAAY;AAAA,cA+GzC,kBAAA,GAAsB,KAAA,EAAO,2BAAA,EAA6B,SAAA,aAAiB,MAAA,CAAA,MAAA;;;;;;;;;;;cA+B3E,2BAAA,GAA+B,KAAA;EAAA,SAAkB,SAAA;AAAA,MAAsB,mBAElF;AAAA,cAEW,2BAAA,GAA+B,KAAA;EAAA,SAAkB,SAAA;AAAA,MAAwC,KAAA,CAAA,KAAA,CAAA,gBAAA"}
1
+ {"version":3,"file":"chunking.d.mts","names":[],"sources":["../src/chunking.ts"],"mappings":";;;;;KASY,2BAAA;EAAA,SACD,OAAA;EAAA,SACA,UAAA;EAAA,SACA,OAAA;EAAA,SACA,SAAA;EAAA,SACA,QAAA,GAAW,iBAAiB;AAAA;AAAA,KAG3B,mBAAA;EAAA,SACD,KAAA,GACP,KAAA,EAAO,2BAAA,KACJ,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,cAAA,GAAiB,sBAAA;AAAA;AAAA,cACnD,qBAAA;cAEY,gBAAA,SAAyB,qBAErC;AAAA,cAEY,yBAAA,GAA6B,IAAY;AAAA,cA+GzC,kBAAA,GAAsB,KAAA,EAAO,2BAAA,EAA6B,SAAA,aAAiB,MAAA,CAAA,MAAA;;;;;;;;;;;cA+B3E,2BAAA,GAA+B,KAAA;EAAA,SAAkB,SAAA;AAAA,MAAsB,mBAElF;AAAA,cAEW,2BAAA,GAA+B,KAAA;EAAA,SAAkB,SAAA;AAAA,MAAwC,KAAA,CAAA,KAAA,CAAA,gBAAA"}
package/dist/chunking.mjs CHANGED
@@ -77,7 +77,7 @@ const chunkKnowledgeText = (input, maxTokens) => Effect.gen(function* () {
77
77
  if (contents.length === 0) return yield* Effect.fail(new KnowledgeChunkingError({ message: "No chunks produced" }));
78
78
  return contents.map((content, position) => ({
79
79
  id: `${input.documentId}:chunk:${position}`,
80
- collectionId: input.collectionId,
80
+ scopeId: input.scopeId,
81
81
  documentId: input.documentId,
82
82
  content,
83
83
  position,
@@ -1 +1 @@
1
- {"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type { KnowledgeChunk, KnowledgeMetadata } from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n readonly collectionId: string\n readonly documentId: string\n readonly content: string\n readonly maxTokens?: number\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n readonly chunk: (\n input: ChunkKnowledgeDocumentInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n text\n .replace(/\\r\\n/g, '\\n')\n .replace(/[\\t ]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()\n\nconst splitSentences = (text: string) =>\n text\n .split(PARAGRAPH_BREAK_PATTERN)\n .flatMap(part => {\n if (part.length === 0) {\n return []\n }\n\n if (/^\\n{2,}$/.test(part)) {\n return ['\\n\\n']\n }\n\n return part.match(SENTENCE_PATTERN) ?? [part]\n })\n .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n const tokens = encode(text)\n return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n const words = text.match(WORD_PATTERN) ?? []\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n for (const word of words) {\n const wordTokenCount = countTokens(word)\n\n if (wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n chunks.push(...splitEncodedTokens(word, maxTokens))\n current = ''\n currentTokenCount = 0\n continue\n }\n\n if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n current = word\n currentTokenCount = wordTokenCount\n continue\n }\n\n current = `${current}${word}`\n currentTokenCount += wordTokenCount\n }\n\n const finalChunk = current.trim()\n if (finalChunk.length > 0) {\n chunks.push(finalChunk)\n }\n\n return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n units.flatMap(unit =>\n countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n const pushCurrent = () => {\n const content = current.trim()\n if (content.length > 0) {\n chunks.push(content)\n }\n }\n\n for (const unit of units) {\n const unitTokenCount = countTokens(unit)\n\n if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n pushCurrent()\n current = unit\n currentTokenCount = unitTokenCount\n continue\n }\n\n current = `${current}${unit}`\n currentTokenCount += unitTokenCount\n }\n\n pushCurrent()\n return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n Effect.gen(function* () {\n if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n )\n }\n\n const sanitized = sanitizeText(input.content)\n if (sanitized.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'Cannot chunk empty content' }))\n }\n\n const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n const contents = buildChunkContents(units, maxTokens)\n\n if (contents.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n }\n\n return contents.map((content, position) => ({\n id: `${input.documentId}:chunk:${position}`,\n collectionId: input.collectionId,\n documentId: input.documentId,\n content,\n position,\n tokenCount: countTokens(content),\n metadata: input.metadata\n })) satisfies ReadonlyArray<KnowledgeChunk>\n })\n\nexport const makeDefaultKnowledgeChunker = (input: { readonly maxTokens: number }): KnowledgeChunkerApi => ({\n chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (input: { readonly maxTokens: number } = { maxTokens: 512 }) =>\n Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAKA,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,eAAe;AAgBrB,IAAa,mBAAb,cAAsC,QAAQ,QAA+C,EAC3F,sCACF,EAAE,CAAC;AAEH,MAAa,6BAA6B,SAAiB,YAAY,IAAI;AAE3E,MAAM,gBAAgB,SACpB,KACG,QAAQ,SAAS,IAAI,EACrB,QAAQ,WAAW,GAAG,EACtB,QAAQ,WAAW,MAAM,EACzB,KAAK;AAEV,MAAM,kBAAkB,SACtB,KACG,MAAM,uBAAuB,EAC7B,SAAQ,SAAQ;CACf,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAGV,IAAI,WAAW,KAAK,IAAI,GACtB,OAAO,CAAC,MAAM;CAGhB,OAAO,KAAK,MAAM,gBAAgB,KAAK,CAAC,IAAI;AAC9C,CAAC,EACA,QAAO,SAAQ,KAAK,SAAS,CAAC;AAEnC,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,SAAS,OAAO,IAAI;CAC1B,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,KAAK,OAAO,SAAS,SAAS,EAAE,IAAI,GAAG,UACtE,OAAO,OAAO,MAAM,QAAQ,WAAW,QAAQ,YAAY,SAAS,CAAC,EAAE,KAAK,CAC9E,EAAE,QAAO,UAAS,MAAM,SAAS,CAAC;AACpC;AAEA,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,QAAQ,KAAK,MAAM,YAAY,KAAK,CAAC;CAC3C,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,iBAAiB,WAAW;GAC9B,MAAM,eAAe,QAAQ,KAAK;GAClC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,OAAO,KAAK,GAAG,mBAAmB,MAAM,SAAS,CAAC;GAClD,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,MAAM,eAAe,QAAQ,KAAK;GAClC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,MAAM,aAAa,QAAQ,KAAK;CAChC,IAAI,WAAW,SAAS,GACtB,OAAO,KAAK,UAAU;CAGxB,OAAO;AACT;AAEA,MAAM,oBAAoB,OAA8B,cACtD,MAAM,SAAQ,SACZ,YAAY,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,mBAAmB,MAAM,SAAS,CAC9E;AAEF,MAAM,sBAAsB,OAA8B,cAAsB;CAC9E,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,MAAM,oBAAoB;EACxB,MAAM,UAAU,QAAQ,KAAK;EAC7B,IAAI,QAAQ,SAAS,GACnB,OAAO,KAAK,OAAO;CAEvB;CAEA,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,YAAY;GACZ,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,YAAY;CACZ,OAAO;AACT;AAEA,MAAa,sBAAsB,OAAoC,cACrE,OAAO,IAAI,aAAa;CACtB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6CAA6C,CAAC,CACtF;CAGF,MAAM,YAAY,aAAa,MAAM,OAAO;CAC5C,IAAI,UAAU,WAAW,GACvB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,6BAA6B,CAAC,CAAC;CAIjG,MAAM,WAAW,mBADH,iBAAiB,eAAe,SAAS,GAAG,SAClB,GAAG,SAAS;CAEpD,IAAI,SAAS,WAAW,GACtB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,qBAAqB,CAAC,CAAC;CAGzF,OAAO,SAAS,KAAK,SAAS,cAAc;EAC1C,IAAI,GAAG,MAAM,WAAW,SAAS;EACjC,cAAc,MAAM;EACpB,YAAY,MAAM;EAClB;EACA;EACA,YAAY,YAAY,OAAO;EAC/B,UAAU,MAAM;CAClB,EAAE;AACJ,CAAC;AAEH,MAAa,+BAA+B,WAAgE,EAC1G,QAAO,aAAY,mBAAmB,UAAU,SAAS,aAAa,MAAM,SAAS,EACvF;AAEA,MAAa,+BAA+B,QAAwC,EAAE,WAAW,IAAI,MACnG,MAAM,QAAQ,kBAAkB,4BAA4B,KAAK,CAAC"}
1
+ {"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type { KnowledgeChunk, KnowledgeMetadata } from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly content: string\n readonly maxTokens?: number\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n readonly chunk: (\n input: ChunkKnowledgeDocumentInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n text\n .replace(/\\r\\n/g, '\\n')\n .replace(/[\\t ]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()\n\nconst splitSentences = (text: string) =>\n text\n .split(PARAGRAPH_BREAK_PATTERN)\n .flatMap(part => {\n if (part.length === 0) {\n return []\n }\n\n if (/^\\n{2,}$/.test(part)) {\n return ['\\n\\n']\n }\n\n return part.match(SENTENCE_PATTERN) ?? [part]\n })\n .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n const tokens = encode(text)\n return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n const words = text.match(WORD_PATTERN) ?? []\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n for (const word of words) {\n const wordTokenCount = countTokens(word)\n\n if (wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n chunks.push(...splitEncodedTokens(word, maxTokens))\n current = ''\n currentTokenCount = 0\n continue\n }\n\n if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n current = word\n currentTokenCount = wordTokenCount\n continue\n }\n\n current = `${current}${word}`\n currentTokenCount += wordTokenCount\n }\n\n const finalChunk = current.trim()\n if (finalChunk.length > 0) {\n chunks.push(finalChunk)\n }\n\n return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n units.flatMap(unit =>\n countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n const pushCurrent = () => {\n const content = current.trim()\n if (content.length > 0) {\n chunks.push(content)\n }\n }\n\n for (const unit of units) {\n const unitTokenCount = countTokens(unit)\n\n if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n pushCurrent()\n current = unit\n currentTokenCount = unitTokenCount\n continue\n }\n\n current = `${current}${unit}`\n currentTokenCount += unitTokenCount\n }\n\n pushCurrent()\n return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n Effect.gen(function* () {\n if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n )\n }\n\n const sanitized = sanitizeText(input.content)\n if (sanitized.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'Cannot chunk empty content' }))\n }\n\n const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n const contents = buildChunkContents(units, maxTokens)\n\n if (contents.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n }\n\n return contents.map((content, position) => ({\n id: `${input.documentId}:chunk:${position}`,\n scopeId: input.scopeId,\n documentId: input.documentId,\n content,\n position,\n tokenCount: countTokens(content),\n metadata: input.metadata\n })) satisfies ReadonlyArray<KnowledgeChunk>\n })\n\nexport const makeDefaultKnowledgeChunker = (input: { readonly maxTokens: number }): KnowledgeChunkerApi => ({\n chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (input: { readonly maxTokens: number } = { maxTokens: 512 }) =>\n Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAKA,MAAM,mBAAmB;AACzB,MAAM,0BAA0B;AAChC,MAAM,eAAe;AAgBrB,IAAa,mBAAb,cAAsC,QAAQ,QAA+C,EAC3F,sCACF,EAAE,CAAC;AAEH,MAAa,6BAA6B,SAAiB,YAAY,IAAI;AAE3E,MAAM,gBAAgB,SACpB,KACG,QAAQ,SAAS,IAAI,EACrB,QAAQ,WAAW,GAAG,EACtB,QAAQ,WAAW,MAAM,EACzB,KAAK;AAEV,MAAM,kBAAkB,SACtB,KACG,MAAM,uBAAuB,EAC7B,SAAQ,SAAQ;CACf,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAGV,IAAI,WAAW,KAAK,IAAI,GACtB,OAAO,CAAC,MAAM;CAGhB,OAAO,KAAK,MAAM,gBAAgB,KAAK,CAAC,IAAI;AAC9C,CAAC,EACA,QAAO,SAAQ,KAAK,SAAS,CAAC;AAEnC,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,SAAS,OAAO,IAAI;CAC1B,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,KAAK,OAAO,SAAS,SAAS,EAAE,IAAI,GAAG,UACtE,OAAO,OAAO,MAAM,QAAQ,WAAW,QAAQ,YAAY,SAAS,CAAC,EAAE,KAAK,CAC9E,EAAE,QAAO,UAAS,MAAM,SAAS,CAAC;AACpC;AAEA,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,QAAQ,KAAK,MAAM,YAAY,KAAK,CAAC;CAC3C,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,iBAAiB,WAAW;GAC9B,MAAM,eAAe,QAAQ,KAAK;GAClC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,OAAO,KAAK,GAAG,mBAAmB,MAAM,SAAS,CAAC;GAClD,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,MAAM,eAAe,QAAQ,KAAK;GAClC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,MAAM,aAAa,QAAQ,KAAK;CAChC,IAAI,WAAW,SAAS,GACtB,OAAO,KAAK,UAAU;CAGxB,OAAO;AACT;AAEA,MAAM,oBAAoB,OAA8B,cACtD,MAAM,SAAQ,SACZ,YAAY,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,mBAAmB,MAAM,SAAS,CAC9E;AAEF,MAAM,sBAAsB,OAA8B,cAAsB;CAC9E,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,MAAM,oBAAoB;EACxB,MAAM,UAAU,QAAQ,KAAK;EAC7B,IAAI,QAAQ,SAAS,GACnB,OAAO,KAAK,OAAO;CAEvB;CAEA,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,YAAY;GACZ,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,YAAY;CACZ,OAAO;AACT;AAEA,MAAa,sBAAsB,OAAoC,cACrE,OAAO,IAAI,aAAa;CACtB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6CAA6C,CAAC,CACtF;CAGF,MAAM,YAAY,aAAa,MAAM,OAAO;CAC5C,IAAI,UAAU,WAAW,GACvB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,6BAA6B,CAAC,CAAC;CAIjG,MAAM,WAAW,mBADH,iBAAiB,eAAe,SAAS,GAAG,SAClB,GAAG,SAAS;CAEpD,IAAI,SAAS,WAAW,GACtB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,qBAAqB,CAAC,CAAC;CAGzF,OAAO,SAAS,KAAK,SAAS,cAAc;EAC1C,IAAI,GAAG,MAAM,WAAW,SAAS;EACjC,SAAS,MAAM;EACf,YAAY,MAAM;EAClB;EACA;EACA,YAAY,YAAY,OAAO;EAC/B,UAAU,MAAM;CAClB,EAAE;AACJ,CAAC;AAEH,MAAa,+BAA+B,WAAgE,EAC1G,QAAO,aAAY,mBAAmB,UAAU,SAAS,aAAa,MAAM,SAAS,EACvF;AAEA,MAAa,+BAA+B,QAAwC,EAAE,WAAW,IAAI,MACnG,MAAM,QAAQ,kBAAkB,4BAA4B,KAAK,CAAC"}
@@ -1,16 +1,11 @@
1
- import { KnowledgeRecord } from "./records.mjs";
2
- import { KnowledgeRepresentation } from "./representations.mjs";
1
+ import { KnowledgeDocument } from "./documents.mjs";
3
2
 
4
3
  //#region src/context.d.ts
5
- type KnowledgeContextItem = {
6
- readonly record: KnowledgeRecord;
7
- readonly representation?: KnowledgeRepresentation;
8
- };
9
4
  type BuildKnowledgeContextInput = {
10
- readonly items: ReadonlyArray<KnowledgeContextItem>;
5
+ readonly documents: ReadonlyArray<KnowledgeDocument>;
11
6
  readonly maxCharacters: number;
12
7
  };
13
8
  declare const buildKnowledgeContext: (input: BuildKnowledgeContextInput) => string;
14
9
  //#endregion
15
- export { BuildKnowledgeContextInput, KnowledgeContextItem, buildKnowledgeContext };
10
+ export { BuildKnowledgeContextInput, buildKnowledgeContext };
16
11
  //# sourceMappingURL=context.d.mts.map
@@ -1 +1 @@
1
- {"version":3,"file":"context.d.mts","names":[],"sources":["../src/context.ts"],"mappings":";;;;KAIY,oBAAA;EAAA,SACD,MAAA,EAAQ,eAAA;EAAA,SACR,cAAA,GAAiB,uBAAuB;AAAA;AAAA,KAGvC,0BAAA;EAAA,SACD,KAAA,EAAO,aAAa,CAAC,oBAAA;EAAA,SACrB,aAAA;AAAA;AAAA,cAeE,qBAAA,GAAyB,KAAiC,EAA1B,0BAA0B"}
1
+ {"version":3,"file":"context.d.mts","names":[],"sources":["../src/context.ts"],"mappings":";;;KAGY,0BAAA;EAAA,SACD,SAAA,EAAW,aAAa,CAAC,iBAAA;EAAA,SACzB,aAAA;AAAA;AAAA,cAuBE,qBAAA,GAAyB,KAAiC,EAA1B,0BAA0B"}