@yolk-sdk/knowledge 0.1.0-canary.123 → 0.1.0-canary.124
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent.d.mts +11 -15
- package/dist/agent.d.mts.map +1 -1
- package/dist/agent.mjs +1 -2
- package/dist/agent.mjs.map +1 -1
- package/dist/chunking.d.mts +8 -11
- package/dist/chunking.d.mts.map +1 -1
- package/dist/chunking.mjs +3 -3
- package/dist/chunking.mjs.map +1 -1
- package/dist/context.d.mts +2 -6
- package/dist/context.d.mts.map +1 -1
- package/dist/context.mjs +1 -2
- package/dist/context.mjs.map +1 -1
- package/dist/documents.d.mts +39 -43
- package/dist/documents.d.mts.map +1 -1
- package/dist/documents.mjs +1 -2
- package/dist/embeddings.d.mts +4 -7
- package/dist/embeddings.d.mts.map +1 -1
- package/dist/embeddings.mjs +1 -2
- package/dist/embeddings.mjs.map +1 -1
- package/dist/errors.d.mts +18 -21
- package/dist/errors.d.mts.map +1 -1
- package/dist/errors.mjs +1 -2
- package/dist/errors.mjs.map +1 -1
- package/dist/extraction.d.mts +4 -7
- package/dist/extraction.d.mts.map +1 -1
- package/dist/extraction.mjs +1 -2
- package/dist/extraction.mjs.map +1 -1
- package/dist/files.d.mts +5 -8
- package/dist/files.d.mts.map +1 -1
- package/dist/files.mjs +1 -2
- package/dist/files.mjs.map +1 -1
- package/dist/index.d.mts +2 -2
- package/dist/index.mjs +2 -2
- package/dist/ingestion.d.mts +14 -18
- package/dist/ingestion.d.mts.map +1 -1
- package/dist/ingestion.mjs +4 -5
- package/dist/ingestion.mjs.map +1 -1
- package/dist/search.d.mts +10 -14
- package/dist/search.d.mts.map +1 -1
- package/dist/search.mjs +2 -3
- package/dist/search.mjs.map +1 -1
- package/dist/store.d.mts +20 -23
- package/dist/store.d.mts.map +1 -1
- package/dist/store.mjs +1 -2
- package/dist/store.mjs.map +1 -1
- package/dist/summarization.d.mts +6 -9
- package/dist/summarization.d.mts.map +1 -1
- package/dist/summarization.mjs +1 -2
- package/dist/summarization.mjs.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -5,11 +5,11 @@ Domain-free knowledge document/source/file/context, ingestion, hybrid search, an
|
|
|
5
5
|
## Install
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
pnpm add @yolk-sdk/knowledge@canary @yolk-sdk/agent@canary effect@4.
|
|
8
|
+
pnpm add @yolk-sdk/knowledge@canary @yolk-sdk/agent@canary effect@4.1.2
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
Canary APIs are unstable. Keep all `@yolk-sdk/*` packages on the same version.
|
|
12
|
-
Use the SDK's matching Effect version (`4.
|
|
12
|
+
Use the SDK's matching Effect version (`4.1.2`) in host code.
|
|
13
13
|
Published package metadata requires Node.js 22+.
|
|
14
14
|
|
|
15
15
|
## Model
|
package/dist/agent.d.mts
CHANGED
|
@@ -3,11 +3,9 @@ import { Effect } from "effect";
|
|
|
3
3
|
import * as Schema from "effect/Schema";
|
|
4
4
|
import { ToolError } from "@yolk-sdk/agent/loop";
|
|
5
5
|
import { ToolRegistration } from "@yolk-sdk/agent/tools";
|
|
6
|
-
|
|
7
|
-
//#region src/agent.d.ts
|
|
8
6
|
/** Declared output of `makeKnowledgeLookupTool`; `structuredContent` is its JSON encoding
|
|
9
7
|
* (`Schema.toCodecJson`: document dates become ISO strings). */
|
|
10
|
-
declare const KnowledgeLookupOutput: Schema.Union<readonly [Schema.Struct<{
|
|
8
|
+
export declare const KnowledgeLookupOutput: Schema.Union<readonly [Schema.Struct<{
|
|
11
9
|
readonly operation: Schema.Literal<"search">;
|
|
12
10
|
readonly results: Schema.$Array<Schema.Struct<{
|
|
13
11
|
readonly document: Schema.Struct<{
|
|
@@ -50,9 +48,9 @@ declare const KnowledgeLookupOutput: Schema.Union<readonly [Schema.Struct<{
|
|
|
50
48
|
readonly updatedAt: Schema.DateTimeUtc;
|
|
51
49
|
}>;
|
|
52
50
|
}>]>;
|
|
53
|
-
type KnowledgeLookupOutput = typeof KnowledgeLookupOutput.Type;
|
|
51
|
+
export type KnowledgeLookupOutput = typeof KnowledgeLookupOutput.Type;
|
|
54
52
|
/** Declared output of `makeKnowledgeManageTool`; `structuredContent` is its JSON encoding. */
|
|
55
|
-
declare const KnowledgeManageOutput: Schema.Struct<{
|
|
53
|
+
export declare const KnowledgeManageOutput: Schema.Struct<{
|
|
56
54
|
readonly operation: Schema.Literals<readonly ["upsert", "set_availability", "rename_slug", "delete"]>;
|
|
57
55
|
readonly document: Schema.Struct<{
|
|
58
56
|
readonly id: Schema.String;
|
|
@@ -60,24 +58,24 @@ declare const KnowledgeManageOutput: Schema.Struct<{
|
|
|
60
58
|
readonly title: Schema.String;
|
|
61
59
|
}>;
|
|
62
60
|
}>;
|
|
63
|
-
type KnowledgeManageOutput = typeof KnowledgeManageOutput.Type;
|
|
64
|
-
type KnowledgeTarget = {
|
|
61
|
+
export type KnowledgeManageOutput = typeof KnowledgeManageOutput.Type;
|
|
62
|
+
export type KnowledgeTarget = {
|
|
65
63
|
readonly slug: string;
|
|
66
64
|
readonly scope?: string;
|
|
67
65
|
};
|
|
68
|
-
type KnowledgeLookupResult = {
|
|
66
|
+
export type KnowledgeLookupResult = {
|
|
69
67
|
readonly document: KnowledgeDocument;
|
|
70
68
|
readonly score?: number;
|
|
71
69
|
readonly context?: ReadonlyArray<{
|
|
72
70
|
readonly content: string;
|
|
73
71
|
}>;
|
|
74
72
|
};
|
|
75
|
-
type KnowledgeSavedDocument = {
|
|
73
|
+
export type KnowledgeSavedDocument = {
|
|
76
74
|
readonly id: string;
|
|
77
75
|
readonly slug: string;
|
|
78
76
|
readonly title: string;
|
|
79
77
|
};
|
|
80
|
-
type KnowledgeLookupHandlers<Context> = {
|
|
78
|
+
export type KnowledgeLookupHandlers<Context> = {
|
|
81
79
|
readonly search: (input: {
|
|
82
80
|
readonly context: Context;
|
|
83
81
|
readonly query: string;
|
|
@@ -91,7 +89,7 @@ type KnowledgeLookupHandlers<Context> = {
|
|
|
91
89
|
readonly target?: KnowledgeTarget;
|
|
92
90
|
}) => Effect.Effect<KnowledgeDocument, ToolError>;
|
|
93
91
|
};
|
|
94
|
-
type KnowledgeManageHandlers<Context> = {
|
|
92
|
+
export type KnowledgeManageHandlers<Context> = {
|
|
95
93
|
readonly upsert: (input: {
|
|
96
94
|
readonly context: Context;
|
|
97
95
|
readonly target: KnowledgeTarget;
|
|
@@ -116,14 +114,12 @@ type KnowledgeManageHandlers<Context> = {
|
|
|
116
114
|
readonly target: KnowledgeTarget;
|
|
117
115
|
}) => Effect.Effect<KnowledgeSavedDocument, ToolError>;
|
|
118
116
|
};
|
|
119
|
-
declare const makeKnowledgeLookupTool: <Context>(handlers: KnowledgeLookupHandlers<Context>, options?: {
|
|
117
|
+
export declare const makeKnowledgeLookupTool: <Context>(handlers: KnowledgeLookupHandlers<Context>, options?: {
|
|
120
118
|
readonly name?: string;
|
|
121
119
|
readonly description?: string;
|
|
122
120
|
}) => ToolRegistration<Context>;
|
|
123
|
-
declare const makeKnowledgeManageTool: <Context>(handlers: KnowledgeManageHandlers<Context>, options?: {
|
|
121
|
+
export declare const makeKnowledgeManageTool: <Context>(handlers: KnowledgeManageHandlers<Context>, options?: {
|
|
124
122
|
readonly name?: string;
|
|
125
123
|
readonly description?: string;
|
|
126
124
|
}) => ToolRegistration<Context>;
|
|
127
|
-
//#endregion
|
|
128
|
-
export { KnowledgeLookupHandlers, KnowledgeLookupOutput, KnowledgeLookupResult, KnowledgeManageHandlers, KnowledgeManageOutput, KnowledgeSavedDocument, KnowledgeTarget, makeKnowledgeLookupTool, makeKnowledgeManageTool };
|
|
129
125
|
//# sourceMappingURL=agent.d.mts.map
|
package/dist/agent.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.mts","names":[],"sources":["../src/agent.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"agent.d.mts","names":[],"sources":["../src/agent.ts"],"mappings":";;;;;;;qBA6Ea,uBAAqB,OAAA,gBAAA,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAWtB,+BAA+B,sBAAsB;;qBAGpD,uBAAqB,OAAA;;;;;;;;YAKtB,+BAA+B,sBAAsB;YAqBrD;WACD;WACA;;YAGC;WACD,UAAU;WACV;WACA,UAAU;aAAyB;;;YAGlC;WACD;WACA;WACA;;YAGC,wBAAwB;WACzB,SAAS;aACP,SAAS;aACT;aACA;aACA;aACA;QACL,OAAO,OAAO,cAAc,wBAAwB;WACjD,MAAM;aACJ,SAAS;aACT;aACA,SAAS;QACd,OAAO,OAAO,mBAAmB;;YAG7B,wBAAwB;WACzB,SAAS;aACP,SAAS;aACT,QAAQ;aACR;aACA;aACA;aACA;aACA,cAAc;QACnB,OAAO,OAAO,wBAAwB;WACnC,kBAAkB;aAChB,SAAS;aACT,QAAQ;aACR,cAAc;QACnB,OAAO,OAAO,wBAAwB;WACnC,aAAa;aACX,SAAS;aACT,QAAQ;aACR;QACL,OAAO,OAAO,wBAAwB;WACnC,SAAS;aACP,SAAS;aACT,QAAQ;QACb,OAAO,OAAO,wBAAwB;;qBA2FjC,0BAA2B,SAAO,UACnC,wBAAwB,UAAQ;WACtB;WAAwB;MAC3C,iBAAiB;qBA0DP,0BAA2B,SAAO,UACnC,wBAAwB,UAAQ;WACtB;WAAwB;MAC3C,iBAAiB"}
|
package/dist/agent.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import * as Schema from "effect/Schema";
|
|
|
4
4
|
import { ToolError } from "@yolk-sdk/agent/loop";
|
|
5
5
|
import { ToolResult } from "@yolk-sdk/agent/protocol";
|
|
6
6
|
import { makeTool, withToolArgumentsErrorHint } from "@yolk-sdk/agent/tools";
|
|
7
|
-
|
|
7
|
+
// src/agent.ts
|
|
8
8
|
const OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString));
|
|
9
9
|
const OptionalPositiveInteger = Schema.optional(Schema.NullOr(PositiveInteger));
|
|
10
10
|
const OptionalNonNegativeInteger = Schema.optional(Schema.NullOr(NonNegativeInteger));
|
|
@@ -262,7 +262,6 @@ const makeKnowledgeManageTool = (handlers, options = {}) => {
|
|
|
262
262
|
})
|
|
263
263
|
});
|
|
264
264
|
};
|
|
265
|
-
//#endregion
|
|
266
265
|
export { KnowledgeLookupOutput, KnowledgeManageOutput, makeKnowledgeLookupTool, makeKnowledgeManageTool };
|
|
267
266
|
|
|
268
267
|
//# sourceMappingURL=agent.mjs.map
|
package/dist/agent.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.mjs","names":[],"sources":["../src/agent.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport * as Schema from 'effect/Schema'\nimport { ToolError } from '@yolk-sdk/agent/loop'\nimport { ToolResult } from '@yolk-sdk/agent/protocol'\nimport { makeTool, withToolArgumentsErrorHint, type ToolRegistration } from '@yolk-sdk/agent/tools'\nimport {\n KnowledgeAvailabilitySchema,\n KnowledgeDocumentSchema,\n NonNegativeInteger,\n NonEmptyTrimmedString,\n PositiveInteger,\n type KnowledgeAvailability,\n type KnowledgeDocument\n} from './documents.ts'\n\nconst OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString))\n\nconst OptionalPositiveInteger = Schema.optional(Schema.NullOr(PositiveInteger))\n\nconst OptionalNonNegativeInteger = Schema.optional(Schema.NullOr(NonNegativeInteger))\n\nconst OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number))\n\nconst KnowledgeTargetParamsSchema = Schema.Struct({\n slug: NonEmptyTrimmedString,\n scope: OptionalNonEmptyTrimmedString\n})\n\nconst KnowledgeLookupParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n query: NonEmptyTrimmedString,\n limit: OptionalPositiveInteger,\n minScore: OptionalNumber,\n contextChunks: OptionalNonNegativeInteger\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n id: OptionalNonEmptyTrimmedString,\n target: Schema.optional(Schema.NullOr(KnowledgeTargetParamsSchema))\n })\n])\n\nconst KnowledgeManageParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('upsert'),\n target: KnowledgeTargetParamsSchema,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('set_availability'),\n target: KnowledgeTargetParamsSchema,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('rename_slug'),\n target: KnowledgeTargetParamsSchema,\n nextSlug: NonEmptyTrimmedString\n }),\n Schema.Struct({\n operation: Schema.Literal('delete'),\n target: KnowledgeTargetParamsSchema\n })\n])\n\nconst KnowledgeLookupResultSchema = Schema.Struct({\n document: KnowledgeDocumentSchema,\n score: Schema.optionalKey(Schema.Number),\n context: Schema.optionalKey(Schema.Array(Schema.Struct({ content: Schema.String })))\n})\n\n/** Declared output of `makeKnowledgeLookupTool`; `structuredContent` is its JSON encoding\n * (`Schema.toCodecJson`: document dates become ISO strings). */\nexport const KnowledgeLookupOutput = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n results: Schema.Array(KnowledgeLookupResultSchema)\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n document: KnowledgeDocumentSchema\n })\n])\n\nexport type KnowledgeLookupOutput = typeof KnowledgeLookupOutput.Type\n\n/** Declared output of `makeKnowledgeManageTool`; `structuredContent` is its JSON encoding. */\nexport const KnowledgeManageOutput = Schema.Struct({\n operation: Schema.Literals(['upsert', 'set_availability', 'rename_slug', 'delete']),\n document: Schema.Struct({ id: Schema.String, slug: Schema.String, title: Schema.String })\n})\n\nexport type KnowledgeManageOutput = typeof KnowledgeManageOutput.Type\n\nconst encodeLookupOutput = Schema.encodeEffect(Schema.toCodecJson(KnowledgeLookupOutput))\n\nconst encodeManageOutput = Schema.encodeEffect(Schema.toCodecJson(KnowledgeManageOutput))\n\n/** A handler result that does not encode to the declared output is a host bug, not a model\n * mistake: fail with a safe message (no values). */\nconst outputError = (tool: string) => () =>\n new ToolError({\n tool,\n message: `${tool} produced a result that does not match its output schema.`,\n cause: 'execution'\n })\n\ntype KnowledgeTargetParams = typeof KnowledgeTargetParamsSchema.Type\n\ntype KnowledgeLookupParams = typeof KnowledgeLookupParams.Type\n\ntype KnowledgeManageParams = typeof KnowledgeManageParams.Type\n\nexport type KnowledgeTarget = {\n readonly slug: string\n readonly scope?: string\n}\n\nexport type KnowledgeLookupResult = {\n readonly document: KnowledgeDocument\n readonly score?: number\n readonly context?: ReadonlyArray<{ readonly content: string }>\n}\n\nexport type KnowledgeSavedDocument = {\n readonly id: string\n readonly slug: string\n readonly title: string\n}\n\nexport type KnowledgeLookupHandlers<Context> = {\n readonly search: (input: {\n readonly context: Context\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n }) => Effect.Effect<ReadonlyArray<KnowledgeLookupResult>, ToolError>\n readonly get: (input: {\n readonly context: Context\n readonly id?: string\n readonly target?: KnowledgeTarget\n }) => Effect.Effect<KnowledgeDocument, ToolError>\n}\n\nexport type KnowledgeManageHandlers<Context> = {\n readonly upsert: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly title: string\n readonly purpose: string\n readonly origin: string\n readonly content: string\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly setAvailability: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly renameSlug: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly nextSlug: string\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly delete: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n}\n\nconst optionalValue = <Value>(value: Value | null | undefined) => value ?? undefined\n\nconst targetFromParams = (target: KnowledgeTargetParams): KnowledgeTarget =>\n target.scope === null || target.scope === undefined\n ? { slug: target.slug }\n : { slug: target.slug, scope: target.scope }\n\nconst optionalTargetFromParams = (target: KnowledgeTargetParams | null | undefined) =>\n target === null || target === undefined ? undefined : targetFromParams(target)\n\nconst formatDocument = (document: KnowledgeDocument) =>\n [\n `# ${document.title}`,\n `document_id: ${document.id}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `availability: ${document.availability}`,\n `status: ${document.status}`,\n '',\n document.content\n ].join('\\n')\n\nconst formatSearchResults = (results: ReadonlyArray<KnowledgeLookupResult>) =>\n results.length === 0\n ? 'No knowledge matches found.'\n : results\n .map(result =>\n [\n `## ${result.document.title}`,\n `document_id: ${result.document.id}`,\n `slug: ${result.document.slug}`,\n `purpose: ${result.document.purpose}`,\n `origin: ${result.document.origin}`,\n `availability: ${result.document.availability}`,\n result.score === undefined ? undefined : `score: ${result.score}`,\n '',\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.document.content\n ]\n .filter(line => line !== undefined)\n .join('\\n')\n )\n .join('\\n\\n')\n\nconst formatSaved = (verb: string, document: KnowledgeSavedDocument) =>\n [verb, `title: ${document.title}`, `slug: ${document.slug}`, `document_id: ${document.id}`].join(\n '\\n'\n )\n\ntype KnowledgeLookupResultOutput = {\n readonly document: KnowledgeDocument\n score?: number\n context?: ReadonlyArray<{ readonly content: string }>\n}\n\nconst lookupResultOutput = (result: KnowledgeLookupResult) => {\n const output: KnowledgeLookupResultOutput = { document: result.document }\n\n if (result.score !== undefined) output.score = result.score\n\n if (result.context !== undefined) {\n output.context = result.context.map(chunk => ({ content: chunk.content }))\n }\n\n return output\n}\n\nconst savedResult = (input: {\n readonly name: string\n readonly callId: string\n readonly verb: string\n readonly operation: KnowledgeManageOutput['operation']\n readonly document: KnowledgeSavedDocument\n}) =>\n encodeManageOutput({\n operation: input.operation,\n document: { id: input.document.id, slug: input.document.slug, title: input.document.title }\n }).pipe(\n Effect.mapError(outputError(input.name)),\n Effect.map(structuredContent =>\n ToolResult.make({\n toolCallId: input.callId,\n content: formatSaved(input.verb, input.document),\n structuredContent\n })\n )\n )\n\nexport const makeKnowledgeLookupTool = <Context>(\n handlers: KnowledgeLookupHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_lookup'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.',\n parameters: KnowledgeLookupParams,\n output: KnowledgeLookupOutput,\n access: 'read',\n invalidParamsMessage: error =>\n withToolArgumentsErrorHint(\n `Invalid knowledge lookup arguments: ${error instanceof Error ? error.message : String(error)}`,\n error\n ),\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n if (params.operation === 'search') {\n const results = yield* handlers.search({\n context,\n query: params.query,\n limit: optionalValue(params.limit),\n minScore: optionalValue(params.minScore),\n contextChunks: optionalValue(params.contextChunks)\n })\n\n const structuredContent = yield* encodeLookupOutput({\n operation: 'search',\n results: results.map(lookupResultOutput)\n }).pipe(Effect.mapError(outputError(name)))\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSearchResults(results),\n structuredContent\n })\n }\n\n const document = yield* handlers.get({\n context,\n id: optionalValue(params.id),\n target: optionalTargetFromParams(params.target)\n })\n\n const structuredContent = yield* encodeLookupOutput({ operation: 'get', document }).pipe(\n Effect.mapError(outputError(name))\n )\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatDocument(document),\n structuredContent\n })\n })\n })\n}\n\nexport const makeKnowledgeManageTool = <Context>(\n handlers: KnowledgeManageHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_manage'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.',\n parameters: KnowledgeManageParams,\n output: KnowledgeManageOutput,\n access: 'write',\n invalidParamsMessage: error =>\n withToolArgumentsErrorHint(\n `Invalid knowledge manage arguments: ${error instanceof Error ? error.message : String(error)}`,\n error\n ),\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n switch (params.operation) {\n case 'upsert': {\n const document = yield* handlers.upsert({\n context,\n target: targetFromParams(params.target),\n title: params.title,\n purpose: params.purpose,\n origin: params.origin,\n content: params.content,\n availability: params.availability\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge upserted',\n operation: 'upsert',\n document\n })\n }\n\n case 'set_availability': {\n const document = yield* handlers.setAvailability({\n context,\n target: targetFromParams(params.target),\n availability: params.availability\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge availability updated',\n operation: 'set_availability',\n document\n })\n }\n\n case 'rename_slug': {\n const document = yield* handlers.renameSlug({\n context,\n target: targetFromParams(params.target),\n nextSlug: params.nextSlug\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge slug renamed',\n operation: 'rename_slug',\n document\n })\n }\n\n case 'delete': {\n const document = yield* handlers.delete({\n context,\n target: targetFromParams(params.target)\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge deleted',\n operation: 'delete',\n document\n })\n }\n }\n\n return yield* Effect.fail(\n new ToolError({\n tool: call.name,\n message: 'Unsupported knowledge operation',\n cause: 'validation'\n })\n )\n })\n })\n}\n"],"mappings":";;;;;;;AAeA,MAAM,gCAAgC,OAAO,SAAS,OAAO,OAAO,qBAAqB,CAAC;AAE1F,MAAM,0BAA0B,OAAO,SAAS,OAAO,OAAO,eAAe,CAAC;AAE9E,MAAM,6BAA6B,OAAO,SAAS,OAAO,OAAO,kBAAkB,CAAC;AAEpF,MAAM,iBAAiB,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC;AAEnE,MAAM,8BAA8B,OAAO,OAAO;CAChD,MAAM;CACN,OAAO;AACT,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM,CACzC,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,OAAO;CACP,OAAO;CACP,UAAU;CACV,eAAe;AACjB,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,IAAI;CACJ,QAAQ,OAAO,SAAS,OAAO,OAAO,2BAA2B,CAAC;AACpE,CAAC,CACH,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM;CACzC,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;EACR,OAAO;EACP,SAAS;EACT,QAAQ;EACR,SAAS;EACT,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,kBAAkB;EAC5C,QAAQ;EACR,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,aAAa;EACvC,QAAQ;EACR,UAAU;CACZ,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;CACV,CAAC;AACH,CAAC;AAED,MAAM,8BAA8B,OAAO,OAAO;CAChD,UAAU;CACV,OAAO,OAAO,YAAY,OAAO,MAAM;CACvC,SAAS,OAAO,YAAY,OAAO,MAAM,OAAO,OAAO,EAAE,SAAS,OAAO,OAAO,CAAC,CAAC,CAAC;AACrF,CAAC;;;AAID,MAAa,wBAAwB,OAAO,MAAM,CAChD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,SAAS,OAAO,MAAM,2BAA2B;AACnD,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,UAAU;AACZ,CAAC,CACH,CAAC;;AAKD,MAAa,wBAAwB,OAAO,OAAO;CACjD,WAAW,OAAO,SAAS;EAAC;EAAU;EAAoB;EAAe;CAAQ,CAAC;CAClF,UAAU,OAAO,OAAO;EAAE,IAAI,OAAO;EAAQ,MAAM,OAAO;EAAQ,OAAO,OAAO;CAAO,CAAC;AAC1F,CAAC;AAID,MAAM,qBAAqB,OAAO,aAAa,OAAO,YAAY,qBAAqB,CAAC;AAExF,MAAM,qBAAqB,OAAO,aAAa,OAAO,YAAY,qBAAqB,CAAC;;;AAIxF,MAAM,eAAe,eACnB,IAAI,UAAU;CACZ;CACA,SAAS,GAAG,KAAK;CACjB,OAAO;AACT,CAAC;AAkEH,MAAM,iBAAwB,UAAoC,SAAS,KAAA;AAE3E,MAAM,oBAAoB,WACxB,OAAO,UAAU,QAAQ,OAAO,UAAU,KAAA,IACtC,EAAE,MAAM,OAAO,KAAK,IACpB;CAAE,MAAM,OAAO;CAAM,OAAO,OAAO;AAAM;AAE/C,MAAM,4BAA4B,WAChC,WAAW,QAAQ,WAAW,KAAA,IAAY,KAAA,IAAY,iBAAiB,MAAM;AAE/E,MAAM,kBAAkB,aACtB;CACE,KAAK,SAAS;CACd,gBAAgB,SAAS;CACzB,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,WAAW,SAAS;CACpB;CACA,SAAS;AACX,EAAE,KAAK,IAAI;AAEb,MAAM,uBAAuB,YAC3B,QAAQ,WAAW,IACf,gCACA,QACG,KAAI,WACH;CACE,MAAM,OAAO,SAAS;CACtB,gBAAgB,OAAO,SAAS;CAChC,SAAS,OAAO,SAAS;CACzB,YAAY,OAAO,SAAS;CAC5B,WAAW,OAAO,SAAS;CAC3B,iBAAiB,OAAO,SAAS;CACjC,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,UAAU,OAAO;CAC1D;CACA,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,SAAS;AAC9E,EACG,QAAO,SAAQ,SAAS,KAAA,CAAS,EACjC,KAAK,IAAI,CACd,EACC,KAAK,MAAM;AAEpB,MAAM,eAAe,MAAc,aACjC;CAAC;CAAM,UAAU,SAAS;CAAS,SAAS,SAAS;CAAQ,gBAAgB,SAAS;AAAI,EAAE,KAC1F,IACF;AAQF,MAAM,sBAAsB,WAAkC;CAC5D,MAAM,SAAsC,EAAE,UAAU,OAAO,SAAS;CAExE,IAAI,OAAO,UAAU,KAAA,GAAW,OAAO,QAAQ,OAAO;CAEtD,IAAI,OAAO,YAAY,KAAA,GACrB,OAAO,UAAU,OAAO,QAAQ,KAAI,WAAU,EAAE,SAAS,MAAM,QAAQ,EAAE;CAG3E,OAAO;AACT;AAEA,MAAM,eAAe,UAOnB,mBAAmB;CACjB,WAAW,MAAM;CACjB,UAAU;EAAE,IAAI,MAAM,SAAS;EAAI,MAAM,MAAM,SAAS;EAAM,OAAO,MAAM,SAAS;CAAM;AAC5F,CAAC,EAAE,KACD,OAAO,SAAS,YAAY,MAAM,IAAI,CAAC,GACvC,OAAO,KAAI,sBACT,WAAW,KAAK;CACd,YAAY,MAAM;CAClB,SAAS,YAAY,MAAM,MAAM,MAAM,QAAQ;CAC/C;AACF,CAAC,CACH,CACF;AAEF,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAC9B,MAAM,OAAO,QAAQ,QAAQ;CAE7B,OAAO,SAAS;EACd;EACA,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,QAAQ;EACR,uBAAsB,UACpB,2BACE,uCAAuC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAC5F,KACF;EACF,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,IAAI,OAAO,cAAc,UAAU;IACjC,MAAM,UAAU,OAAO,SAAS,OAAO;KACrC;KACA,OAAO,OAAO;KACd,OAAO,cAAc,OAAO,KAAK;KACjC,UAAU,cAAc,OAAO,QAAQ;KACvC,eAAe,cAAc,OAAO,aAAa;IACnD,CAAC;IAED,MAAM,oBAAoB,OAAO,mBAAmB;KAClD,WAAW;KACX,SAAS,QAAQ,IAAI,kBAAkB;IACzC,CAAC,EAAE,KAAK,OAAO,SAAS,YAAY,IAAI,CAAC,CAAC;IAE1C,OAAO,WAAW,KAAK;KACrB,YAAY,KAAK;KACjB,SAAS,oBAAoB,OAAO;KACpC;IACF,CAAC;GACH;GAEA,MAAM,WAAW,OAAO,SAAS,IAAI;IACnC;IACA,IAAI,cAAc,OAAO,EAAE;IAC3B,QAAQ,yBAAyB,OAAO,MAAM;GAChD,CAAC;GAED,MAAM,oBAAoB,OAAO,mBAAmB;IAAE,WAAW;IAAO;GAAS,CAAC,EAAE,KAClF,OAAO,SAAS,YAAY,IAAI,CAAC,CACnC;GAEA,OAAO,WAAW,KAAK;IACrB,YAAY,KAAK;IACjB,SAAS,eAAe,QAAQ;IAChC;GACF,CAAC;EACH,CAAC;CACL,CAAC;AACH;AAEA,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAC9B,MAAM,OAAO,QAAQ,QAAQ;CAE7B,OAAO,SAAS;EACd;EACA,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,QAAQ;EACR,uBAAsB,UACpB,2BACE,uCAAuC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAC5F,KACF;EACF,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,QAAQ,OAAO,WAAf;IACE,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,OAAO,OAAO;MACd,SAAS,OAAO;MAChB,QAAQ,OAAO;MACf,SAAS,OAAO;MAChB,cAAc,OAAO;KACvB,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;IAEA,KAAK,oBAAoB;KACvB,MAAM,WAAW,OAAO,SAAS,gBAAgB;MAC/C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,cAAc,OAAO;KACvB,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;IAEA,KAAK,eAAe;KAClB,MAAM,WAAW,OAAO,SAAS,WAAW;MAC1C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,UAAU,OAAO;KACnB,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;IAEA,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;KACxC,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;GACF;GAEA,OAAO,OAAO,OAAO,KACnB,IAAI,UAAU;IACZ,MAAM,KAAK;IACX,SAAS;IACT,OAAO;GACT,CAAC,CACH;EACF,CAAC;CACL,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"agent.mjs","names":[],"sources":["../src/agent.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport * as Schema from 'effect/Schema'\nimport { ToolError } from '@yolk-sdk/agent/loop'\nimport { ToolResult } from '@yolk-sdk/agent/protocol'\nimport { makeTool, withToolArgumentsErrorHint, type ToolRegistration } from '@yolk-sdk/agent/tools'\nimport {\n KnowledgeAvailabilitySchema,\n KnowledgeDocumentSchema,\n NonNegativeInteger,\n NonEmptyTrimmedString,\n PositiveInteger,\n type KnowledgeAvailability,\n type KnowledgeDocument\n} from './documents.ts'\n\nconst OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString))\n\nconst OptionalPositiveInteger = Schema.optional(Schema.NullOr(PositiveInteger))\n\nconst OptionalNonNegativeInteger = Schema.optional(Schema.NullOr(NonNegativeInteger))\n\nconst OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number))\n\nconst KnowledgeTargetParamsSchema = Schema.Struct({\n slug: NonEmptyTrimmedString,\n scope: OptionalNonEmptyTrimmedString\n})\n\nconst KnowledgeLookupParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n query: NonEmptyTrimmedString,\n limit: OptionalPositiveInteger,\n minScore: OptionalNumber,\n contextChunks: OptionalNonNegativeInteger\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n id: OptionalNonEmptyTrimmedString,\n target: Schema.optional(Schema.NullOr(KnowledgeTargetParamsSchema))\n })\n])\n\nconst KnowledgeManageParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('upsert'),\n target: KnowledgeTargetParamsSchema,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('set_availability'),\n target: KnowledgeTargetParamsSchema,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('rename_slug'),\n target: KnowledgeTargetParamsSchema,\n nextSlug: NonEmptyTrimmedString\n }),\n Schema.Struct({\n operation: Schema.Literal('delete'),\n target: KnowledgeTargetParamsSchema\n })\n])\n\nconst KnowledgeLookupResultSchema = Schema.Struct({\n document: KnowledgeDocumentSchema,\n score: Schema.optionalKey(Schema.Number),\n context: Schema.optionalKey(Schema.Array(Schema.Struct({ content: Schema.String })))\n})\n\n/** Declared output of `makeKnowledgeLookupTool`; `structuredContent` is its JSON encoding\n * (`Schema.toCodecJson`: document dates become ISO strings). */\nexport const KnowledgeLookupOutput = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n results: Schema.Array(KnowledgeLookupResultSchema)\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n document: KnowledgeDocumentSchema\n })\n])\n\nexport type KnowledgeLookupOutput = typeof KnowledgeLookupOutput.Type\n\n/** Declared output of `makeKnowledgeManageTool`; `structuredContent` is its JSON encoding. */\nexport const KnowledgeManageOutput = Schema.Struct({\n operation: Schema.Literals(['upsert', 'set_availability', 'rename_slug', 'delete']),\n document: Schema.Struct({ id: Schema.String, slug: Schema.String, title: Schema.String })\n})\n\nexport type KnowledgeManageOutput = typeof KnowledgeManageOutput.Type\n\nconst encodeLookupOutput = Schema.encodeEffect(Schema.toCodecJson(KnowledgeLookupOutput))\n\nconst encodeManageOutput = Schema.encodeEffect(Schema.toCodecJson(KnowledgeManageOutput))\n\n/** A handler result that does not encode to the declared output is a host bug, not a model\n * mistake: fail with a safe message (no values). */\nconst outputError = (tool: string) => () =>\n new ToolError({\n tool,\n message: `${tool} produced a result that does not match its output schema.`,\n cause: 'execution'\n })\n\ntype KnowledgeTargetParams = typeof KnowledgeTargetParamsSchema.Type\n\ntype KnowledgeLookupParams = typeof KnowledgeLookupParams.Type\n\ntype KnowledgeManageParams = typeof KnowledgeManageParams.Type\n\nexport type KnowledgeTarget = {\n readonly slug: string\n readonly scope?: string\n}\n\nexport type KnowledgeLookupResult = {\n readonly document: KnowledgeDocument\n readonly score?: number\n readonly context?: ReadonlyArray<{ readonly content: string }>\n}\n\nexport type KnowledgeSavedDocument = {\n readonly id: string\n readonly slug: string\n readonly title: string\n}\n\nexport type KnowledgeLookupHandlers<Context> = {\n readonly search: (input: {\n readonly context: Context\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n }) => Effect.Effect<ReadonlyArray<KnowledgeLookupResult>, ToolError>\n readonly get: (input: {\n readonly context: Context\n readonly id?: string\n readonly target?: KnowledgeTarget\n }) => Effect.Effect<KnowledgeDocument, ToolError>\n}\n\nexport type KnowledgeManageHandlers<Context> = {\n readonly upsert: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly title: string\n readonly purpose: string\n readonly origin: string\n readonly content: string\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly setAvailability: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly renameSlug: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly nextSlug: string\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly delete: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n}\n\nconst optionalValue = <Value>(value: Value | null | undefined) => value ?? undefined\n\nconst targetFromParams = (target: KnowledgeTargetParams): KnowledgeTarget =>\n target.scope === null || target.scope === undefined\n ? { slug: target.slug }\n : { slug: target.slug, scope: target.scope }\n\nconst optionalTargetFromParams = (target: KnowledgeTargetParams | null | undefined) =>\n target === null || target === undefined ? undefined : targetFromParams(target)\n\nconst formatDocument = (document: KnowledgeDocument) =>\n [\n `# ${document.title}`,\n `document_id: ${document.id}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `availability: ${document.availability}`,\n `status: ${document.status}`,\n '',\n document.content\n ].join('\\n')\n\nconst formatSearchResults = (results: ReadonlyArray<KnowledgeLookupResult>) =>\n results.length === 0\n ? 'No knowledge matches found.'\n : results\n .map(result =>\n [\n `## ${result.document.title}`,\n `document_id: ${result.document.id}`,\n `slug: ${result.document.slug}`,\n `purpose: ${result.document.purpose}`,\n `origin: ${result.document.origin}`,\n `availability: ${result.document.availability}`,\n result.score === undefined ? undefined : `score: ${result.score}`,\n '',\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.document.content\n ]\n .filter(line => line !== undefined)\n .join('\\n')\n )\n .join('\\n\\n')\n\nconst formatSaved = (verb: string, document: KnowledgeSavedDocument) =>\n [verb, `title: ${document.title}`, `slug: ${document.slug}`, `document_id: ${document.id}`].join(\n '\\n'\n )\n\ntype KnowledgeLookupResultOutput = {\n readonly document: KnowledgeDocument\n score?: number\n context?: ReadonlyArray<{ readonly content: string }>\n}\n\nconst lookupResultOutput = (result: KnowledgeLookupResult) => {\n const output: KnowledgeLookupResultOutput = { document: result.document }\n\n if (result.score !== undefined) output.score = result.score\n\n if (result.context !== undefined) {\n output.context = result.context.map(chunk => ({ content: chunk.content }))\n }\n\n return output\n}\n\nconst savedResult = (input: {\n readonly name: string\n readonly callId: string\n readonly verb: string\n readonly operation: KnowledgeManageOutput['operation']\n readonly document: KnowledgeSavedDocument\n}) =>\n encodeManageOutput({\n operation: input.operation,\n document: { id: input.document.id, slug: input.document.slug, title: input.document.title }\n }).pipe(\n Effect.mapError(outputError(input.name)),\n Effect.map(structuredContent =>\n ToolResult.make({\n toolCallId: input.callId,\n content: formatSaved(input.verb, input.document),\n structuredContent\n })\n )\n )\n\nexport const makeKnowledgeLookupTool = <Context>(\n handlers: KnowledgeLookupHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_lookup'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.',\n parameters: KnowledgeLookupParams,\n output: KnowledgeLookupOutput,\n access: 'read',\n invalidParamsMessage: error =>\n withToolArgumentsErrorHint(\n `Invalid knowledge lookup arguments: ${error instanceof Error ? error.message : String(error)}`,\n error\n ),\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n if (params.operation === 'search') {\n const results = yield* handlers.search({\n context,\n query: params.query,\n limit: optionalValue(params.limit),\n minScore: optionalValue(params.minScore),\n contextChunks: optionalValue(params.contextChunks)\n })\n\n const structuredContent = yield* encodeLookupOutput({\n operation: 'search',\n results: results.map(lookupResultOutput)\n }).pipe(Effect.mapError(outputError(name)))\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSearchResults(results),\n structuredContent\n })\n }\n\n const document = yield* handlers.get({\n context,\n id: optionalValue(params.id),\n target: optionalTargetFromParams(params.target)\n })\n\n const structuredContent = yield* encodeLookupOutput({ operation: 'get', document }).pipe(\n Effect.mapError(outputError(name))\n )\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatDocument(document),\n structuredContent\n })\n })\n })\n}\n\nexport const makeKnowledgeManageTool = <Context>(\n handlers: KnowledgeManageHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_manage'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.',\n parameters: KnowledgeManageParams,\n output: KnowledgeManageOutput,\n access: 'write',\n invalidParamsMessage: error =>\n withToolArgumentsErrorHint(\n `Invalid knowledge manage arguments: ${error instanceof Error ? error.message : String(error)}`,\n error\n ),\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n switch (params.operation) {\n case 'upsert': {\n const document = yield* handlers.upsert({\n context,\n target: targetFromParams(params.target),\n title: params.title,\n purpose: params.purpose,\n origin: params.origin,\n content: params.content,\n availability: params.availability\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge upserted',\n operation: 'upsert',\n document\n })\n }\n\n case 'set_availability': {\n const document = yield* handlers.setAvailability({\n context,\n target: targetFromParams(params.target),\n availability: params.availability\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge availability updated',\n operation: 'set_availability',\n document\n })\n }\n\n case 'rename_slug': {\n const document = yield* handlers.renameSlug({\n context,\n target: targetFromParams(params.target),\n nextSlug: params.nextSlug\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge slug renamed',\n operation: 'rename_slug',\n document\n })\n }\n\n case 'delete': {\n const document = yield* handlers.delete({\n context,\n target: targetFromParams(params.target)\n })\n\n return yield* savedResult({\n name,\n callId: call.id,\n verb: 'Knowledge deleted',\n operation: 'delete',\n document\n })\n }\n }\n\n return yield* Effect.fail(\n new ToolError({\n tool: call.name,\n message: 'Unsupported knowledge operation',\n cause: 'validation'\n })\n )\n })\n })\n}\n"],"mappings":";;;;;;;AAeA,MAAM,gCAAgC,OAAO,SAAS,OAAO,OAAO,qBAAqB,CAAC;AAE1F,MAAM,0BAA0B,OAAO,SAAS,OAAO,OAAO,eAAe,CAAC;AAE9E,MAAM,6BAA6B,OAAO,SAAS,OAAO,OAAO,kBAAkB,CAAC;AAEpF,MAAM,iBAAiB,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC;AAEnE,MAAM,8BAA8B,OAAO,OAAO;CAChD,MAAM;CACN,OAAO;AACT,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM,CACzC,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,OAAO;CACP,OAAO;CACP,UAAU;CACV,eAAe;AACjB,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,IAAI;CACJ,QAAQ,OAAO,SAAS,OAAO,OAAO,2BAA2B,CAAC;AACpE,CAAC,CACH,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM;CACzC,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;EACR,OAAO;EACP,SAAS;EACT,QAAQ;EACR,SAAS;EACT,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,kBAAkB;EAC5C,QAAQ;EACR,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,aAAa;EACvC,QAAQ;EACR,UAAU;CACZ,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;CACV,CAAC;AACH,CAAC;AAED,MAAM,8BAA8B,OAAO,OAAO;CAChD,UAAU;CACV,OAAO,OAAO,YAAY,OAAO,MAAM;CACvC,SAAS,OAAO,YAAY,OAAO,MAAM,OAAO,OAAO,EAAE,SAAS,OAAO,OAAO,CAAC,CAAC,CAAC;AACrF,CAAC;;;AAID,MAAa,wBAAwB,OAAO,MAAM,CAChD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,SAAS,OAAO,MAAM,2BAA2B;AACnD,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,UAAU;AACZ,CAAC,CACH,CAAC;;AAKD,MAAa,wBAAwB,OAAO,OAAO;CACjD,WAAW,OAAO,SAAS;EAAC;EAAU;EAAoB;EAAe;CAAQ,CAAC;CAClF,UAAU,OAAO,OAAO;EAAE,IAAI,OAAO;EAAQ,MAAM,OAAO;EAAQ,OAAO,OAAO;CAAO,CAAC;AAC1F,CAAC;AAID,MAAM,qBAAqB,OAAO,aAAa,OAAO,YAAY,qBAAqB,CAAC;AAExF,MAAM,qBAAqB,OAAO,aAAa,OAAO,YAAY,qBAAqB,CAAC;;;AAIxF,MAAM,eAAe,eACnB,IAAI,UAAU;CACZ;CACA,SAAS,GAAG,KAAK;CACjB,OAAO;AACT,CAAC;AAkEH,MAAM,iBAAwB,UAAoC,SAAS,KAAA;AAE3E,MAAM,oBAAoB,WACxB,OAAO,UAAU,QAAQ,OAAO,UAAU,KAAA,IACtC,EAAE,MAAM,OAAO,KAAK,IACpB;CAAE,MAAM,OAAO;CAAM,OAAO,OAAO;AAAM;AAE/C,MAAM,4BAA4B,WAChC,WAAW,QAAQ,WAAW,KAAA,IAAY,KAAA,IAAY,iBAAiB,MAAM;AAE/E,MAAM,kBAAkB,aACtB;CACE,KAAK,SAAS;CACd,gBAAgB,SAAS;CACzB,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,WAAW,SAAS;CACpB;CACA,SAAS;AACX,CAAC,CAAC,KAAK,IAAI;AAEb,MAAM,uBAAuB,YAC3B,QAAQ,WAAW,IACf,gCACA,QACG,KAAI,WACH;CACE,MAAM,OAAO,SAAS;CACtB,gBAAgB,OAAO,SAAS;CAChC,SAAS,OAAO,SAAS;CACzB,YAAY,OAAO,SAAS;CAC5B,WAAW,OAAO,SAAS;CAC3B,iBAAiB,OAAO,SAAS;CACjC,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,UAAU,OAAO;CAC1D;CACA,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,CAAC,CAAC,KAAK,MAAM,KAAK,OAAO,SAAS;AAC9E,CAAC,CACE,QAAO,SAAQ,SAAS,KAAA,CAAS,CAAC,CAClC,KAAK,IAAI,CACd,CAAC,CACA,KAAK,MAAM;AAEpB,MAAM,eAAe,MAAc,aACjC;CAAC;CAAM,UAAU,SAAS;CAAS,SAAS,SAAS;CAAQ,gBAAgB,SAAS;AAAI,CAAC,CAAC,KAC1F,IACF;AAQF,MAAM,sBAAsB,WAAkC;CAC5D,MAAM,SAAsC,EAAE,UAAU,OAAO,SAAS;CAExE,IAAI,OAAO,UAAU,KAAA,GAAW,OAAO,QAAQ,OAAO;CAEtD,IAAI,OAAO,YAAY,KAAA,GACrB,OAAO,UAAU,OAAO,QAAQ,KAAI,WAAU,EAAE,SAAS,MAAM,QAAQ,EAAE;CAG3E,OAAO;AACT;AAEA,MAAM,eAAe,UAOnB,mBAAmB;CACjB,WAAW,MAAM;CACjB,UAAU;EAAE,IAAI,MAAM,SAAS;EAAI,MAAM,MAAM,SAAS;EAAM,OAAO,MAAM,SAAS;CAAM;AAC5F,CAAC,CAAC,CAAC,KACD,OAAO,SAAS,YAAY,MAAM,IAAI,CAAC,GACvC,OAAO,KAAI,sBACT,WAAW,KAAK;CACd,YAAY,MAAM;CAClB,SAAS,YAAY,MAAM,MAAM,MAAM,QAAQ;CAC/C;AACF,CAAC,CACH,CACF;AAEF,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAC9B,MAAM,OAAO,QAAQ,QAAQ;CAE7B,OAAO,SAAS;EACd;EACA,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,QAAQ;EACR,uBAAsB,UACpB,2BACE,uCAAuC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAC5F,KACF;EACF,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,IAAI,OAAO,cAAc,UAAU;IACjC,MAAM,UAAU,OAAO,SAAS,OAAO;KACrC;KACA,OAAO,OAAO;KACd,OAAO,cAAc,OAAO,KAAK;KACjC,UAAU,cAAc,OAAO,QAAQ;KACvC,eAAe,cAAc,OAAO,aAAa;IACnD,CAAC;IAED,MAAM,oBAAoB,OAAO,mBAAmB;KAClD,WAAW;KACX,SAAS,QAAQ,IAAI,kBAAkB;IACzC,CAAC,CAAC,CAAC,KAAK,OAAO,SAAS,YAAY,IAAI,CAAC,CAAC;IAE1C,OAAO,WAAW,KAAK;KACrB,YAAY,KAAK;KACjB,SAAS,oBAAoB,OAAO;KACpC;IACF,CAAC;GACH;GAEA,MAAM,WAAW,OAAO,SAAS,IAAI;IACnC;IACA,IAAI,cAAc,OAAO,EAAE;IAC3B,QAAQ,yBAAyB,OAAO,MAAM;GAChD,CAAC;GAED,MAAM,oBAAoB,OAAO,mBAAmB;IAAE,WAAW;IAAO;GAAS,CAAC,CAAC,CAAC,KAClF,OAAO,SAAS,YAAY,IAAI,CAAC,CACnC;GAEA,OAAO,WAAW,KAAK;IACrB,YAAY,KAAK;IACjB,SAAS,eAAe,QAAQ;IAChC;GACF,CAAC;EACH,CAAC;CACL,CAAC;AACH;AAEA,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAC9B,MAAM,OAAO,QAAQ,QAAQ;CAE7B,OAAO,SAAS;EACd;EACA,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,QAAQ;EACR,uBAAsB,UACpB,2BACE,uCAAuC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,KAC5F,KACF;EACF,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,QAAQ,OAAO,WAAf;IACE,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,OAAO,OAAO;MACd,SAAS,OAAO;MAChB,QAAQ,OAAO;MACf,SAAS,OAAO;MAChB,cAAc,OAAO;KACvB,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;IAEA,KAAK,oBAAoB;KACvB,MAAM,WAAW,OAAO,SAAS,gBAAgB;MAC/C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,cAAc,OAAO;KACvB,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;IAEA,KAAK,eAAe;KAClB,MAAM,WAAW,OAAO,SAAS,WAAW;MAC1C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,UAAU,OAAO;KACnB,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;IAEA,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;KACxC,CAAC;KAED,OAAO,OAAO,YAAY;MACxB;MACA,QAAQ,KAAK;MACb,MAAM;MACN,WAAW;MACX;KACF,CAAC;IACH;GACF;GAEA,OAAO,OAAO,OAAO,KACnB,IAAI,UAAU;IACZ,MAAM,KAAK;IACX,SAAS;IACT,OAAO;GACT,CAAC,CACH;EACF,CAAC;CACL,CAAC;AACH"}
|
package/dist/chunking.d.mts
CHANGED
|
@@ -1,22 +1,20 @@
|
|
|
1
1
|
import { KnowledgeChunk, KnowledgeDocumentId, KnowledgeMetadata, KnowledgeScopeId } from "./documents.mjs";
|
|
2
2
|
import { KnowledgeChunkingError } from "./errors.mjs";
|
|
3
3
|
import { Context, Effect, Layer } from "effect";
|
|
4
|
-
|
|
5
|
-
//#region src/chunking.d.ts
|
|
6
|
-
type ChunkKnowledgeDocumentInput = {
|
|
4
|
+
export type ChunkKnowledgeDocumentInput = {
|
|
7
5
|
readonly scopeId: KnowledgeScopeId;
|
|
8
6
|
readonly documentId: KnowledgeDocumentId;
|
|
9
7
|
readonly content: string;
|
|
10
8
|
readonly maxTokens?: number;
|
|
11
9
|
readonly metadata?: KnowledgeMetadata;
|
|
12
10
|
};
|
|
13
|
-
type KnowledgeChunkerApi = {
|
|
11
|
+
export type KnowledgeChunkerApi = {
|
|
14
12
|
readonly chunk: (input: ChunkKnowledgeDocumentInput) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>;
|
|
15
13
|
};
|
|
16
14
|
declare const KnowledgeChunker_base: Context.ServiceClass<KnowledgeChunker, "@yolk-sdk/knowledge/KnowledgeChunker", KnowledgeChunkerApi>;
|
|
17
|
-
declare class KnowledgeChunker extends KnowledgeChunker_base {}
|
|
18
|
-
declare const countKnowledgeChunkTokens: (text: string) => number;
|
|
19
|
-
declare const chunkKnowledgeText: (input: ChunkKnowledgeDocumentInput, maxTokens: number) => Effect.Effect<{
|
|
15
|
+
export declare class KnowledgeChunker extends KnowledgeChunker_base {}
|
|
16
|
+
export declare const countKnowledgeChunkTokens: (text: string) => number;
|
|
17
|
+
export declare const chunkKnowledgeText: (input: ChunkKnowledgeDocumentInput, maxTokens: number) => Effect.Effect<{
|
|
20
18
|
id: string;
|
|
21
19
|
scopeId: string & import("effect/Brand").Brand<"KnowledgeScopeId">;
|
|
22
20
|
documentId: string & import("effect/Brand").Brand<"KnowledgeDocumentId">;
|
|
@@ -27,12 +25,11 @@ declare const chunkKnowledgeText: (input: ChunkKnowledgeDocumentInput, maxTokens
|
|
|
27
25
|
readonly [x: string]: unknown;
|
|
28
26
|
} | undefined;
|
|
29
27
|
}[], KnowledgeChunkingError, never>;
|
|
30
|
-
declare const makeDefaultKnowledgeChunker: (input: {
|
|
28
|
+
export declare const makeDefaultKnowledgeChunker: (input: {
|
|
31
29
|
readonly maxTokens: number;
|
|
32
30
|
}) => KnowledgeChunkerApi;
|
|
33
|
-
declare const DefaultKnowledgeChunkerLive: (input?: {
|
|
31
|
+
export declare const DefaultKnowledgeChunkerLive: (input?: {
|
|
34
32
|
readonly maxTokens: number;
|
|
35
33
|
}) => Layer.Layer<KnowledgeChunker, never, never>;
|
|
36
|
-
|
|
37
|
-
export { ChunkKnowledgeDocumentInput, DefaultKnowledgeChunkerLive, KnowledgeChunker, KnowledgeChunkerApi, chunkKnowledgeText, countKnowledgeChunkTokens, makeDefaultKnowledgeChunker };
|
|
34
|
+
export {};
|
|
38
35
|
//# sourceMappingURL=chunking.d.mts.map
|
package/dist/chunking.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.mts","names":[],"sources":["../src/chunking.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"chunking.d.mts","names":[],"sources":["../src/chunking.ts"],"mappings":";;;YAgBY;WACD,SAAS;WACT,YAAY;WACZ;WACA;WACA,WAAW;;YAGV;WACD,QACP,OAAO,gCACJ,OAAO,OAAO,cAAc,iBAAiB;;;qBAGvC,yBAAyB;qBAIzB,4BAAyB;qBAqHzB,qBAAkB,OAAW,6BAA2B,sBAAmB,OAAA;;;;;;;;;;KAgCpF;qBAES,8BAA2B;WAC7B;MACP;qBAIS,8BAA2B;WACpB;MAAmB,MAAA,MAAA"}
|
package/dist/chunking.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { KnowledgeChunkingError } from "./errors.mjs";
|
|
2
2
|
import { Context, Effect, Layer } from "effect";
|
|
3
3
|
import { countTokens, decode, encode } from "gpt-tokenizer/encoding/o200k_base";
|
|
4
|
-
|
|
4
|
+
// src/chunking.ts
|
|
5
5
|
const SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:["')\]]+)?\s*|[^.!?]+$/g;
|
|
6
6
|
const PARAGRAPH_BREAK_PATTERN = /(\n{2,})/;
|
|
7
7
|
const WORD_PATTERN = /\S+\s*/g;
|
|
@@ -73,7 +73,8 @@ const chunkKnowledgeText = (input, maxTokens) => Effect.gen(function* () {
|
|
|
73
73
|
if (!Number.isInteger(maxTokens) || maxTokens < 1) return yield* Effect.fail(new KnowledgeChunkingError({ message: "Chunk maxTokens must be a positive integer" }));
|
|
74
74
|
const sanitized = sanitizeText(input.content);
|
|
75
75
|
if (sanitized.length === 0) return yield* Effect.fail(new KnowledgeChunkingError({ message: "Cannot chunk empty content" }));
|
|
76
|
-
const
|
|
76
|
+
const units = splitUnitsBySize(splitSentences(sanitized), maxTokens);
|
|
77
|
+
const contents = buildChunkContents(units, maxTokens);
|
|
77
78
|
if (contents.length === 0) return yield* Effect.fail(new KnowledgeChunkingError({ message: "No chunks produced" }));
|
|
78
79
|
return contents.map((content, position) => ({
|
|
79
80
|
id: `${input.documentId}:chunk:${position}`,
|
|
@@ -87,7 +88,6 @@ const chunkKnowledgeText = (input, maxTokens) => Effect.gen(function* () {
|
|
|
87
88
|
});
|
|
88
89
|
const makeDefaultKnowledgeChunker = (input) => ({ chunk: (document) => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens) });
|
|
89
90
|
const DefaultKnowledgeChunkerLive = (input = { maxTokens: 512 }) => Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input));
|
|
90
|
-
//#endregion
|
|
91
91
|
export { DefaultKnowledgeChunkerLive, KnowledgeChunker, chunkKnowledgeText, countKnowledgeChunkTokens, makeDefaultKnowledgeChunker };
|
|
92
92
|
|
|
93
93
|
//# sourceMappingURL=chunking.mjs.map
|
package/dist/chunking.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type {\n KnowledgeChunk,\n KnowledgeDocumentId,\n KnowledgeMetadata,\n KnowledgeScopeId\n} from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\n\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\n\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n readonly scopeId: KnowledgeScopeId\n readonly documentId: KnowledgeDocumentId\n readonly content: string\n readonly maxTokens?: number\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n readonly chunk: (\n input: ChunkKnowledgeDocumentInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n text\n .replace(/\\r\\n/g, '\\n')\n .replace(/[\\t ]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()\n\nconst splitSentences = (text: string) =>\n text\n .split(PARAGRAPH_BREAK_PATTERN)\n .flatMap(part => {\n if (part.length === 0) {\n return []\n }\n\n if (/^\\n{2,}$/.test(part)) {\n return ['\\n\\n']\n }\n\n return part.match(SENTENCE_PATTERN) ?? [part]\n })\n .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n const tokens = encode(text)\n\n return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n const words = text.match(WORD_PATTERN) ?? []\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n for (const word of words) {\n const wordTokenCount = countTokens(word)\n\n if (wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n chunks.push(...splitEncodedTokens(word, maxTokens))\n current = ''\n currentTokenCount = 0\n continue\n }\n\n if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n current = word\n currentTokenCount = wordTokenCount\n continue\n }\n\n current = `${current}${word}`\n currentTokenCount += wordTokenCount\n }\n\n const finalChunk = current.trim()\n\n if (finalChunk.length > 0) {\n chunks.push(finalChunk)\n }\n\n return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n units.flatMap(unit =>\n countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n const pushCurrent = () => {\n const content = current.trim()\n\n if (content.length > 0) {\n chunks.push(content)\n }\n }\n\n for (const unit of units) {\n const unitTokenCount = countTokens(unit)\n\n if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n pushCurrent()\n current = unit\n currentTokenCount = unitTokenCount\n continue\n }\n\n current = `${current}${unit}`\n currentTokenCount += unitTokenCount\n }\n\n pushCurrent()\n\n return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n Effect.gen(function* () {\n if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n )\n }\n\n const sanitized = sanitizeText(input.content)\n\n if (sanitized.length === 0) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Cannot chunk empty content' })\n )\n }\n\n const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n const contents = buildChunkContents(units, maxTokens)\n\n if (contents.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n }\n\n return contents.map((content, position) => ({\n id: `${input.documentId}:chunk:${position}`,\n scopeId: input.scopeId,\n documentId: input.documentId,\n content,\n position,\n tokenCount: countTokens(content),\n metadata: input.metadata\n })) satisfies ReadonlyArray<KnowledgeChunk>\n })\n\nexport const makeDefaultKnowledgeChunker = (input: {\n readonly maxTokens: number\n}): KnowledgeChunkerApi => ({\n chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (\n input: { readonly maxTokens: number } = { maxTokens: 512 }\n) => Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAUA,MAAM,mBAAmB;AAEzB,MAAM,0BAA0B;AAEhC,MAAM,eAAe;AAgBrB,IAAa,mBAAb,cAAsC,QAAQ,QAA+C,
|
|
1
|
+
{"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type {\n KnowledgeChunk,\n KnowledgeDocumentId,\n KnowledgeMetadata,\n KnowledgeScopeId\n} from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\n\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\n\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n readonly scopeId: KnowledgeScopeId\n readonly documentId: KnowledgeDocumentId\n readonly content: string\n readonly maxTokens?: number\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n readonly chunk: (\n input: ChunkKnowledgeDocumentInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n text\n .replace(/\\r\\n/g, '\\n')\n .replace(/[\\t ]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()\n\nconst splitSentences = (text: string) =>\n text\n .split(PARAGRAPH_BREAK_PATTERN)\n .flatMap(part => {\n if (part.length === 0) {\n return []\n }\n\n if (/^\\n{2,}$/.test(part)) {\n return ['\\n\\n']\n }\n\n return part.match(SENTENCE_PATTERN) ?? [part]\n })\n .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n const tokens = encode(text)\n\n return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n const words = text.match(WORD_PATTERN) ?? []\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n for (const word of words) {\n const wordTokenCount = countTokens(word)\n\n if (wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n chunks.push(...splitEncodedTokens(word, maxTokens))\n current = ''\n currentTokenCount = 0\n continue\n }\n\n if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n current = word\n currentTokenCount = wordTokenCount\n continue\n }\n\n current = `${current}${word}`\n currentTokenCount += wordTokenCount\n }\n\n const finalChunk = current.trim()\n\n if (finalChunk.length > 0) {\n chunks.push(finalChunk)\n }\n\n return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n units.flatMap(unit =>\n countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n const pushCurrent = () => {\n const content = current.trim()\n\n if (content.length > 0) {\n chunks.push(content)\n }\n }\n\n for (const unit of units) {\n const unitTokenCount = countTokens(unit)\n\n if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n pushCurrent()\n current = unit\n currentTokenCount = unitTokenCount\n continue\n }\n\n current = `${current}${unit}`\n currentTokenCount += unitTokenCount\n }\n\n pushCurrent()\n\n return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n Effect.gen(function* () {\n if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n )\n }\n\n const sanitized = sanitizeText(input.content)\n\n if (sanitized.length === 0) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Cannot chunk empty content' })\n )\n }\n\n const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n const contents = buildChunkContents(units, maxTokens)\n\n if (contents.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n }\n\n return contents.map((content, position) => ({\n id: `${input.documentId}:chunk:${position}`,\n scopeId: input.scopeId,\n documentId: input.documentId,\n content,\n position,\n tokenCount: countTokens(content),\n metadata: input.metadata\n })) satisfies ReadonlyArray<KnowledgeChunk>\n })\n\nexport const makeDefaultKnowledgeChunker = (input: {\n readonly maxTokens: number\n}): KnowledgeChunkerApi => ({\n chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (\n input: { readonly maxTokens: number } = { maxTokens: 512 }\n) => Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAUA,MAAM,mBAAmB;AAEzB,MAAM,0BAA0B;AAEhC,MAAM,eAAe;AAgBrB,IAAa,mBAAb,cAAsC,QAAQ,QAA+C,CAAC,CAC5F,sCACF,CAAC,CAAC,CAAC;AAEH,MAAa,6BAA6B,SAAiB,YAAY,IAAI;AAE3E,MAAM,gBAAgB,SACpB,KACG,QAAQ,SAAS,IAAI,CAAC,CACtB,QAAQ,WAAW,GAAG,CAAC,CACvB,QAAQ,WAAW,MAAM,CAAC,CAC1B,KAAK;AAEV,MAAM,kBAAkB,SACtB,KACG,MAAM,uBAAuB,CAAC,CAC9B,SAAQ,SAAQ;CACf,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAGV,IAAI,WAAW,KAAK,IAAI,GACtB,OAAO,CAAC,MAAM;CAGhB,OAAO,KAAK,MAAM,gBAAgB,KAAK,CAAC,IAAI;AAC9C,CAAC,CAAC,CACD,QAAO,SAAQ,KAAK,SAAS,CAAC;AAEnC,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,SAAS,OAAO,IAAI;CAE1B,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,KAAK,OAAO,SAAS,SAAS,EAAE,IAAI,GAAG,UACtE,OAAO,OAAO,MAAM,QAAQ,WAAW,QAAQ,YAAY,SAAS,CAAC,CAAC,CAAC,KAAK,CAC9E,CAAC,CAAC,QAAO,UAAS,MAAM,SAAS,CAAC;AACpC;AAEA,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,QAAQ,KAAK,MAAM,YAAY,KAAK,CAAC;CAC3C,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,iBAAiB,WAAW;GAC9B,MAAM,eAAe,QAAQ,KAAK;GAElC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,OAAO,KAAK,GAAG,mBAAmB,MAAM,SAAS,CAAC;GAClD,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,MAAM,eAAe,QAAQ,KAAK;GAElC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,MAAM,aAAa,QAAQ,KAAK;CAEhC,IAAI,WAAW,SAAS,GACtB,OAAO,KAAK,UAAU;CAGxB,OAAO;AACT;AAEA,MAAM,oBAAoB,OAA8B,cACtD,MAAM,SAAQ,SACZ,YAAY,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,mBAAmB,MAAM,SAAS,CAC9E;AAEF,MAAM,sBAAsB,OAA8B,cAAsB;CAC9E,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,MAAM,oBAAoB;EACxB,MAAM,UAAU,QAAQ,KAAK;EAE7B,IAAI,QAAQ,SAAS,GACnB,OAAO,KAAK,OAAO;CAEvB;CAEA,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,YAAY;GACZ,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,YAAY;CAEZ,OAAO;AACT;AAEA,MAAa,sBAAsB,OAAoC,cACrE,OAAO,IAAI,aAAa;CACtB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6CAA6C,CAAC,CACtF;CAGF,MAAM,YAAY,aAAa,MAAM,OAAO;CAE5C,IAAI,UAAU,WAAW,GACvB,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6BAA6B,CAAC,CACtE;CAGF,MAAM,QAAQ,iBAAiB,eAAe,SAAS,GAAG,SAAS;CACnE,MAAM,WAAW,mBAAmB,OAAO,SAAS;CAEpD,IAAI,SAAS,WAAW,GACtB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,qBAAqB,CAAC,CAAC;CAGzF,OAAO,SAAS,KAAK,SAAS,cAAc;EAC1C,IAAI,GAAG,MAAM,WAAW,SAAS;EACjC,SAAS,MAAM;EACf,YAAY,MAAM;EAClB;EACA;EACA,YAAY,YAAY,OAAO;EAC/B,UAAU,MAAM;CAClB,EAAE;AACJ,CAAC;AAEH,MAAa,+BAA+B,WAEhB,EAC1B,QAAO,aAAY,mBAAmB,UAAU,SAAS,aAAa,MAAM,SAAS,EACvF;AAEA,MAAa,+BACX,QAAwC,EAAE,WAAW,IAAI,MACtD,MAAM,QAAQ,kBAAkB,4BAA4B,KAAK,CAAC"}
|
package/dist/context.d.mts
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
import { KnowledgeDocument } from "./documents.mjs";
|
|
2
|
-
|
|
3
|
-
//#region src/context.d.ts
|
|
4
|
-
type BuildKnowledgeContextInput = {
|
|
2
|
+
export type BuildKnowledgeContextInput = {
|
|
5
3
|
readonly documents: ReadonlyArray<KnowledgeDocument>;
|
|
6
4
|
readonly maxCharacters: number;
|
|
7
5
|
};
|
|
8
|
-
declare const buildKnowledgeContext: (input: BuildKnowledgeContextInput) => string;
|
|
9
|
-
//#endregion
|
|
10
|
-
export { BuildKnowledgeContextInput, buildKnowledgeContext };
|
|
6
|
+
export declare const buildKnowledgeContext: (input: BuildKnowledgeContextInput) => string;
|
|
11
7
|
//# sourceMappingURL=context.d.mts.map
|
package/dist/context.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.d.mts","names":[],"sources":["../src/context.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"context.d.mts","names":[],"sources":["../src/context.ts"],"mappings":";YAGY;WACD,WAAW,cAAc;WACzB;;qBAuBE,wBAAqB,OAAW"}
|
package/dist/context.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Array } from "effect";
|
|
2
|
-
|
|
2
|
+
// src/context.ts
|
|
3
3
|
const truncationMarker = "\n\n[truncated: pinned knowledge exceeded context budget]";
|
|
4
4
|
const documentHeader = (document) => [
|
|
5
5
|
`## ${document.title}`,
|
|
@@ -24,7 +24,6 @@ const buildKnowledgeContext = (input) => {
|
|
|
24
24
|
if (input.maxCharacters <= 55) return context.slice(0, input.maxCharacters).trimEnd();
|
|
25
25
|
return `${context.slice(0, input.maxCharacters - 55).trimEnd()}${truncationMarker}`;
|
|
26
26
|
};
|
|
27
|
-
//#endregion
|
|
28
27
|
export { buildKnowledgeContext };
|
|
29
28
|
|
|
30
29
|
//# sourceMappingURL=context.mjs.map
|
package/dist/context.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.mjs","names":["Arr"],"sources":["../src/context.ts"],"sourcesContent":["import { Array as Arr } from 'effect'\nimport type { KnowledgeDocument } from './documents.ts'\n\nexport type BuildKnowledgeContextInput = {\n readonly documents: ReadonlyArray<KnowledgeDocument>\n readonly maxCharacters: number\n}\n\nconst truncationMarker = '\\n\\n[truncated: pinned knowledge exceeded context budget]'\n\nconst documentHeader = (document: KnowledgeDocument) =>\n [\n `## ${document.title}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `status: ${document.status}`,\n `availability: ${document.availability}`,\n document.reviewedAt === undefined ? undefined : `reviewed_at: ${document.reviewedAt.toString()}`\n ]\n .filter(line => line !== undefined && line.trim().length > 0)\n .join('\\n')\n\nconst documentBody = (document: KnowledgeDocument) =>\n [documentHeader(document), document.summary, document.content]\n .filter(section => section !== undefined && section.trim().length > 0)\n .join('\\n')\n\nexport const buildKnowledgeContext = (input: BuildKnowledgeContextInput) => {\n const sections = Arr.map(input.documents, documentBody).filter(\n section => section.trim().length > 0\n )\n\n const header = '# Pinned knowledge\\nUse this durable knowledge as high-priority context.'\n const body = sections.join('\\n\\n')\n const context = body.length === 0 ? '' : `${header}\\n\\n${body}`\n\n if (context.length <= input.maxCharacters) {\n return context\n }\n\n if (input.maxCharacters <= truncationMarker.length) {\n return context.slice(0, input.maxCharacters).trimEnd()\n }\n\n return `${context.slice(0, input.maxCharacters - truncationMarker.length).trimEnd()}${truncationMarker}`\n}\n"],"mappings":";;AAQA,MAAM,mBAAmB;AAEzB,MAAM,kBAAkB,aACtB;CACE,MAAM,SAAS;CACf,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,SAAS,eAAe,KAAA,IAAY,KAAA,IAAY,gBAAgB,SAAS,WAAW,SAAS;AAC/F,
|
|
1
|
+
{"version":3,"file":"context.mjs","names":["Arr"],"sources":["../src/context.ts"],"sourcesContent":["import { Array as Arr } from 'effect'\nimport type { KnowledgeDocument } from './documents.ts'\n\nexport type BuildKnowledgeContextInput = {\n readonly documents: ReadonlyArray<KnowledgeDocument>\n readonly maxCharacters: number\n}\n\nconst truncationMarker = '\\n\\n[truncated: pinned knowledge exceeded context budget]'\n\nconst documentHeader = (document: KnowledgeDocument) =>\n [\n `## ${document.title}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `status: ${document.status}`,\n `availability: ${document.availability}`,\n document.reviewedAt === undefined ? undefined : `reviewed_at: ${document.reviewedAt.toString()}`\n ]\n .filter(line => line !== undefined && line.trim().length > 0)\n .join('\\n')\n\nconst documentBody = (document: KnowledgeDocument) =>\n [documentHeader(document), document.summary, document.content]\n .filter(section => section !== undefined && section.trim().length > 0)\n .join('\\n')\n\nexport const buildKnowledgeContext = (input: BuildKnowledgeContextInput) => {\n const sections = Arr.map(input.documents, documentBody).filter(\n section => section.trim().length > 0\n )\n\n const header = '# Pinned knowledge\\nUse this durable knowledge as high-priority context.'\n const body = sections.join('\\n\\n')\n const context = body.length === 0 ? '' : `${header}\\n\\n${body}`\n\n if (context.length <= input.maxCharacters) {\n return context\n }\n\n if (input.maxCharacters <= truncationMarker.length) {\n return context.slice(0, input.maxCharacters).trimEnd()\n }\n\n return `${context.slice(0, input.maxCharacters - truncationMarker.length).trimEnd()}${truncationMarker}`\n}\n"],"mappings":";;AAQA,MAAM,mBAAmB;AAEzB,MAAM,kBAAkB,aACtB;CACE,MAAM,SAAS;CACf,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,SAAS,eAAe,KAAA,IAAY,KAAA,IAAY,gBAAgB,SAAS,WAAW,SAAS;AAC/F,CAAC,CACE,QAAO,SAAQ,SAAS,KAAA,KAAa,KAAK,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CAC5D,KAAK,IAAI;AAEd,MAAM,gBAAgB,aACpB;CAAC,eAAe,QAAQ;CAAG,SAAS;CAAS,SAAS;AAAO,CAAC,CAC3D,QAAO,YAAW,YAAY,KAAA,KAAa,QAAQ,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,CACrE,KAAK,IAAI;AAEd,MAAa,yBAAyB,UAAsC;CAC1E,MAAM,WAAWA,MAAI,IAAI,MAAM,WAAW,YAAY,CAAC,CAAC,QACtD,YAAW,QAAQ,KAAK,CAAC,CAAC,SAAS,CACrC;CAEA,MAAM,SAAS;CACf,MAAM,OAAO,SAAS,KAAK,MAAM;CACjC,MAAM,UAAU,KAAK,WAAW,IAAI,KAAK,GAAG,OAAO,MAAM;CAEzD,IAAI,QAAQ,UAAU,MAAM,eAC1B,OAAO;CAGT,IAAI,MAAM,iBAAiB,IACzB,OAAO,QAAQ,MAAM,GAAG,MAAM,aAAa,CAAC,CAAC,QAAQ;CAGvD,OAAO,GAAG,QAAQ,MAAM,GAAG,MAAM,gBAAgB,EAAuB,CAAC,CAAC,QAAQ,IAAI;AACxF"}
|
package/dist/documents.d.mts
CHANGED
|
@@ -1,25 +1,23 @@
|
|
|
1
1
|
import * as Schema from "effect/Schema";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
declare const
|
|
5
|
-
declare const
|
|
6
|
-
|
|
7
|
-
declare const
|
|
8
|
-
type
|
|
9
|
-
declare const
|
|
10
|
-
type
|
|
11
|
-
declare const
|
|
12
|
-
type
|
|
13
|
-
declare const
|
|
14
|
-
type
|
|
15
|
-
declare const
|
|
16
|
-
type KnowledgeDocumentId = typeof KnowledgeDocumentId.Type;
|
|
17
|
-
declare const KnowledgeScopeSchema: Schema.Struct<{
|
|
2
|
+
export declare const NonEmptyTrimmedString: Schema.Trimmed;
|
|
3
|
+
export declare const NonNegativeInteger: Schema.Int;
|
|
4
|
+
export declare const PositiveInteger: Schema.Int;
|
|
5
|
+
export declare const KnowledgeMetadataSchema: Schema.$Record<Schema.String, Schema.Unknown>;
|
|
6
|
+
export type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>;
|
|
7
|
+
export declare const KnowledgeAvailabilitySchema: Schema.Literals<readonly ["pinned", "searchable", "archived"]>;
|
|
8
|
+
export type KnowledgeAvailability = Schema.Schema.Type<typeof KnowledgeAvailabilitySchema>;
|
|
9
|
+
export declare const KnowledgeDocumentStatusSchema: Schema.Literals<readonly ["processing", "ready", "error"]>;
|
|
10
|
+
export type KnowledgeDocumentStatus = Schema.Schema.Type<typeof KnowledgeDocumentStatusSchema>;
|
|
11
|
+
export declare const KnowledgeScopeId: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
12
|
+
export type KnowledgeScopeId = typeof KnowledgeScopeId.Type;
|
|
13
|
+
export declare const KnowledgeDocumentId: Schema.brand<Schema.Trimmed, "KnowledgeDocumentId">;
|
|
14
|
+
export type KnowledgeDocumentId = typeof KnowledgeDocumentId.Type;
|
|
15
|
+
export declare const KnowledgeScopeSchema: Schema.Struct<{
|
|
18
16
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
19
17
|
readonly kind: Schema.optional<Schema.Trimmed>;
|
|
20
18
|
}>;
|
|
21
|
-
type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>;
|
|
22
|
-
declare const KnowledgeDocumentSchema: Schema.Struct<{
|
|
19
|
+
export type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>;
|
|
20
|
+
export declare const KnowledgeDocumentSchema: Schema.Struct<{
|
|
23
21
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeDocumentId">;
|
|
24
22
|
readonly slug: Schema.Trimmed;
|
|
25
23
|
readonly title: Schema.Trimmed;
|
|
@@ -35,8 +33,8 @@ declare const KnowledgeDocumentSchema: Schema.Struct<{
|
|
|
35
33
|
readonly createdAt: Schema.DateTimeUtc;
|
|
36
34
|
readonly updatedAt: Schema.DateTimeUtc;
|
|
37
35
|
}>;
|
|
38
|
-
type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>;
|
|
39
|
-
declare const CreateKnowledgeDocumentInputSchema: Schema.Struct<{
|
|
36
|
+
export type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>;
|
|
37
|
+
export declare const CreateKnowledgeDocumentInputSchema: Schema.Struct<{
|
|
40
38
|
readonly scope: Schema.Struct<{
|
|
41
39
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
42
40
|
readonly kind: Schema.optional<Schema.Trimmed>;
|
|
@@ -50,8 +48,8 @@ declare const CreateKnowledgeDocumentInputSchema: Schema.Struct<{
|
|
|
50
48
|
readonly summary: Schema.optional<Schema.String>;
|
|
51
49
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
52
50
|
}>;
|
|
53
|
-
type CreateKnowledgeDocumentInput = Schema.Schema.Type<typeof CreateKnowledgeDocumentInputSchema>;
|
|
54
|
-
declare const UpdateKnowledgeDocumentInputSchema: Schema.Struct<{
|
|
51
|
+
export type CreateKnowledgeDocumentInput = Schema.Schema.Type<typeof CreateKnowledgeDocumentInputSchema>;
|
|
52
|
+
export declare const UpdateKnowledgeDocumentInputSchema: Schema.Struct<{
|
|
55
53
|
readonly scope: Schema.Struct<{
|
|
56
54
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
57
55
|
readonly kind: Schema.optional<Schema.Trimmed>;
|
|
@@ -69,8 +67,8 @@ declare const UpdateKnowledgeDocumentInputSchema: Schema.Struct<{
|
|
|
69
67
|
readonly reviewedAt: Schema.optional<Schema.DateTimeUtc>;
|
|
70
68
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
71
69
|
}>;
|
|
72
|
-
type UpdateKnowledgeDocumentInput = Schema.Schema.Type<typeof UpdateKnowledgeDocumentInputSchema>;
|
|
73
|
-
declare const KnowledgeFileSchema: Schema.Struct<{
|
|
70
|
+
export type UpdateKnowledgeDocumentInput = Schema.Schema.Type<typeof UpdateKnowledgeDocumentInputSchema>;
|
|
71
|
+
export declare const KnowledgeFileSchema: Schema.Struct<{
|
|
74
72
|
readonly id: Schema.Trimmed;
|
|
75
73
|
readonly documentId: Schema.brand<Schema.Trimmed, "KnowledgeDocumentId">;
|
|
76
74
|
readonly storageKey: Schema.Trimmed;
|
|
@@ -80,8 +78,8 @@ declare const KnowledgeFileSchema: Schema.Struct<{
|
|
|
80
78
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
81
79
|
readonly createdAt: Schema.DateTimeUtc;
|
|
82
80
|
}>;
|
|
83
|
-
type KnowledgeFile = Schema.Schema.Type<typeof KnowledgeFileSchema>;
|
|
84
|
-
declare const KnowledgeChunkSchema: Schema.Struct<{
|
|
81
|
+
export type KnowledgeFile = Schema.Schema.Type<typeof KnowledgeFileSchema>;
|
|
82
|
+
export declare const KnowledgeChunkSchema: Schema.Struct<{
|
|
85
83
|
readonly id: Schema.Trimmed;
|
|
86
84
|
readonly scopeId: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
87
85
|
readonly documentId: Schema.brand<Schema.Trimmed, "KnowledgeDocumentId">;
|
|
@@ -90,31 +88,31 @@ declare const KnowledgeChunkSchema: Schema.Struct<{
|
|
|
90
88
|
readonly tokenCount: Schema.Int;
|
|
91
89
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
92
90
|
}>;
|
|
93
|
-
type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>;
|
|
94
|
-
declare const KnowledgeSearchScope: Schema.TaggedStruct<"KnowledgeScope", {
|
|
91
|
+
export type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>;
|
|
92
|
+
export declare const KnowledgeSearchScope: Schema.TaggedStruct<"KnowledgeScope", {
|
|
95
93
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
96
94
|
}>;
|
|
97
|
-
declare const KnowledgeSearchScopes: Schema.TaggedStruct<"KnowledgeScopes", {
|
|
95
|
+
export declare const KnowledgeSearchScopes: Schema.TaggedStruct<"KnowledgeScopes", {
|
|
98
96
|
readonly ids: Schema.NonEmptyArray<Schema.brand<Schema.Trimmed, "KnowledgeScopeId">>;
|
|
99
97
|
}>;
|
|
100
|
-
declare const KnowledgeSearchScopeSchema: Schema.Union<readonly [Schema.TaggedStruct<"KnowledgeScope", {
|
|
98
|
+
export declare const KnowledgeSearchScopeSchema: Schema.Union<readonly [Schema.TaggedStruct<"KnowledgeScope", {
|
|
101
99
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
102
100
|
}>, Schema.TaggedStruct<"KnowledgeScopes", {
|
|
103
101
|
readonly ids: Schema.NonEmptyArray<Schema.brand<Schema.Trimmed, "KnowledgeScopeId">>;
|
|
104
102
|
}>]>;
|
|
105
|
-
type KnowledgeSearchScope = typeof KnowledgeSearchScopeSchema.Type;
|
|
106
|
-
declare const KnowledgeFileSource: Schema.TaggedStruct<"File", {
|
|
103
|
+
export type KnowledgeSearchScope = typeof KnowledgeSearchScopeSchema.Type;
|
|
104
|
+
export declare const KnowledgeFileSource: Schema.TaggedStruct<"File", {
|
|
107
105
|
readonly ref: Schema.Trimmed;
|
|
108
106
|
readonly name: Schema.optional<Schema.Trimmed>;
|
|
109
107
|
readonly mediaType: Schema.optional<Schema.Trimmed>;
|
|
110
108
|
}>;
|
|
111
|
-
declare const KnowledgeUrlSource: Schema.TaggedStruct<"Url", {
|
|
109
|
+
export declare const KnowledgeUrlSource: Schema.TaggedStruct<"Url", {
|
|
112
110
|
readonly url: Schema.Trimmed;
|
|
113
111
|
}>;
|
|
114
|
-
declare const KnowledgeTextSource: Schema.TaggedStruct<"Text", {
|
|
112
|
+
export declare const KnowledgeTextSource: Schema.TaggedStruct<"Text", {
|
|
115
113
|
readonly label: Schema.optional<Schema.Trimmed>;
|
|
116
114
|
}>;
|
|
117
|
-
declare const KnowledgeSourceSchema: Schema.Union<readonly [Schema.TaggedStruct<"File", {
|
|
115
|
+
export declare const KnowledgeSourceSchema: Schema.Union<readonly [Schema.TaggedStruct<"File", {
|
|
118
116
|
readonly ref: Schema.Trimmed;
|
|
119
117
|
readonly name: Schema.optional<Schema.Trimmed>;
|
|
120
118
|
readonly mediaType: Schema.optional<Schema.Trimmed>;
|
|
@@ -123,8 +121,8 @@ declare const KnowledgeSourceSchema: Schema.Union<readonly [Schema.TaggedStruct<
|
|
|
123
121
|
}>, Schema.TaggedStruct<"Text", {
|
|
124
122
|
readonly label: Schema.optional<Schema.Trimmed>;
|
|
125
123
|
}>]>;
|
|
126
|
-
type KnowledgeSource = typeof KnowledgeSourceSchema.Type;
|
|
127
|
-
declare const IndexedKnowledgeDocumentSchema: Schema.Struct<{
|
|
124
|
+
export type KnowledgeSource = typeof KnowledgeSourceSchema.Type;
|
|
125
|
+
export declare const IndexedKnowledgeDocumentSchema: Schema.Struct<{
|
|
128
126
|
readonly id: Schema.brand<Schema.Trimmed, "KnowledgeDocumentId">;
|
|
129
127
|
readonly scopeId: Schema.brand<Schema.Trimmed, "KnowledgeScopeId">;
|
|
130
128
|
readonly source: Schema.Union<readonly [Schema.TaggedStruct<"File", {
|
|
@@ -145,15 +143,13 @@ declare const IndexedKnowledgeDocumentSchema: Schema.Struct<{
|
|
|
145
143
|
readonly chunkCount: Schema.optional<Schema.Int>;
|
|
146
144
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
147
145
|
}>;
|
|
148
|
-
type IndexedKnowledgeDocument = Schema.Schema.Type<typeof IndexedKnowledgeDocumentSchema>;
|
|
149
|
-
declare const ExtractedKnowledgeDocumentSchema: Schema.Struct<{
|
|
146
|
+
export type IndexedKnowledgeDocument = Schema.Schema.Type<typeof IndexedKnowledgeDocumentSchema>;
|
|
147
|
+
export declare const ExtractedKnowledgeDocumentSchema: Schema.Struct<{
|
|
150
148
|
readonly content: Schema.Trimmed;
|
|
151
149
|
readonly title: Schema.optional<Schema.Trimmed>;
|
|
152
150
|
readonly summary: Schema.optional<Schema.String>;
|
|
153
151
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
154
152
|
}>;
|
|
155
|
-
type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>;
|
|
156
|
-
declare const defaultKnowledgeChunkMaxTokens = 512;
|
|
157
|
-
//#endregion
|
|
158
|
-
export { CreateKnowledgeDocumentInput, CreateKnowledgeDocumentInputSchema, ExtractedKnowledgeDocument, ExtractedKnowledgeDocumentSchema, IndexedKnowledgeDocument, IndexedKnowledgeDocumentSchema, KnowledgeAvailability, KnowledgeAvailabilitySchema, KnowledgeChunk, KnowledgeChunkSchema, KnowledgeDocument, KnowledgeDocumentId, KnowledgeDocumentSchema, KnowledgeDocumentStatus, KnowledgeDocumentStatusSchema, KnowledgeFile, KnowledgeFileSchema, KnowledgeFileSource, KnowledgeMetadata, KnowledgeMetadataSchema, KnowledgeScope, KnowledgeScopeId, KnowledgeScopeSchema, KnowledgeSearchScope, KnowledgeSearchScopeSchema, KnowledgeSearchScopes, KnowledgeSource, KnowledgeSourceSchema, KnowledgeTextSource, KnowledgeUrlSource, NonEmptyTrimmedString, NonNegativeInteger, PositiveInteger, UpdateKnowledgeDocumentInput, UpdateKnowledgeDocumentInputSchema, defaultKnowledgeChunkMaxTokens };
|
|
153
|
+
export type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>;
|
|
154
|
+
export declare const defaultKnowledgeChunkMaxTokens = 512;
|
|
159
155
|
//# sourceMappingURL=documents.d.mts.map
|
package/dist/documents.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"documents.d.mts","names":[],"sources":["../src/documents.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"documents.d.mts","names":[],"sources":["../src/documents.ts"],"mappings":";qBAEa,uBAAqB,OAAA;qBAErB,oBAAkB,OAAA;qBAElB,iBAAe,OAAA;qBAEf,yBAAuB,OAAA,QAAA,OAAA,QAAA,OAAA;YAExB,oBAAoB,OAAO,OAAO,YAAY;qBAE7C,6BAA2B,OAAA;YAE5B,wBAAwB,OAAO,OAAO,YAAY;qBAEjD,+BAA6B,OAAA;YAE9B,0BAA0B,OAAO,OAAO,YAAY;qBAEnD,kBAAgB,OAAA,MAAA,OAAA;YAEjB,0BAA0B,iBAAiB;qBAE1C,qBAAmB,OAAA,MAAA,OAAA;YAEpB,6BAA6B,oBAAoB;qBAEhD,sBAAoB,OAAA;;;;YAKrB,iBAAiB,OAAO,OAAO,YAAY;qBAE1C,yBAAuB,OAAA;;;;;;;;;;;;;;;;YAiBxB,oBAAoB,OAAO,OAAO,YAAY;qBAE7C,oCAAkC,OAAA;;;;;;;;;;;;;;YAYnC,+BAA+B,OAAO,OAAO,YAChD;qBAGI,oCAAkC,OAAA;;;;;;;;;;;;;;;;;;YAgBnC,+BAA+B,OAAO,OAAO,YAChD;qBAGI,qBAAmB,OAAA;;;;;;;;;;YAWpB,gBAAgB,OAAO,OAAO,YAAY;qBAEzC,sBAAoB,OAAA;;;;;;;;;YAUrB,iBAAiB,OAAO,OAAO,YAAY;qBAE1C,sBAAoB,OAAA;;;qBAIpB,uBAAqB,OAAA;;;qBAIrB,4BAA0B,OAAA,gBAAA,OAAA;;;;;YAK3B,8BAA8B,2BAA2B;qBAExD,qBAAmB,OAAA;;;;;qBAMnB,oBAAkB,OAAA;;;qBAIlB,qBAAmB,OAAA;;;qBAInB,uBAAqB,OAAA,gBAAA,OAAA;;;;;;;;;YAMtB,yBAAyB,sBAAsB;qBAE9C,gCAA8B,OAAA;;;;;;;;;;;;;;;;;;;;;YAc/B,2BAA2B,OAAO,OAAO,YAAY;qBAEpD,kCAAgC,OAAA;;;;;;YAOjC,6BAA6B,OAAO,OAAO,YAAY;qBAEtD"}
|
package/dist/documents.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as Schema from "effect/Schema";
|
|
2
|
-
|
|
2
|
+
// src/documents.ts
|
|
3
3
|
const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()));
|
|
4
4
|
const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)));
|
|
5
5
|
const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)));
|
|
@@ -116,7 +116,6 @@ const ExtractedKnowledgeDocumentSchema = Schema.Struct({
|
|
|
116
116
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
117
117
|
});
|
|
118
118
|
const defaultKnowledgeChunkMaxTokens = 512;
|
|
119
|
-
//#endregion
|
|
120
119
|
export { CreateKnowledgeDocumentInputSchema, ExtractedKnowledgeDocumentSchema, IndexedKnowledgeDocumentSchema, KnowledgeAvailabilitySchema, KnowledgeChunkSchema, KnowledgeDocumentId, KnowledgeDocumentSchema, KnowledgeDocumentStatusSchema, KnowledgeFileSchema, KnowledgeFileSource, KnowledgeMetadataSchema, KnowledgeScopeId, KnowledgeScopeSchema, KnowledgeSearchScope, KnowledgeSearchScopeSchema, KnowledgeSearchScopes, KnowledgeSourceSchema, KnowledgeTextSource, KnowledgeUrlSource, NonEmptyTrimmedString, NonNegativeInteger, PositiveInteger, UpdateKnowledgeDocumentInputSchema, defaultKnowledgeChunkMaxTokens };
|
|
121
120
|
|
|
122
121
|
//# sourceMappingURL=documents.mjs.map
|
package/dist/embeddings.d.mts
CHANGED
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
import { KnowledgeEmbeddingError } from "./errors.mjs";
|
|
2
2
|
import { Context, Effect } from "effect";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
type KnowledgeEmbedding = ReadonlyArray<number>;
|
|
6
|
-
type KnowledgeEmbedderApi = {
|
|
3
|
+
export type KnowledgeEmbedding = ReadonlyArray<number>;
|
|
4
|
+
export type KnowledgeEmbedderApi = {
|
|
7
5
|
readonly embedTexts: (texts: ReadonlyArray<string>) => Effect.Effect<ReadonlyArray<KnowledgeEmbedding>, KnowledgeEmbeddingError>;
|
|
8
6
|
readonly embedQuery: (query: string) => Effect.Effect<KnowledgeEmbedding, KnowledgeEmbeddingError>;
|
|
9
7
|
};
|
|
10
8
|
declare const KnowledgeEmbedder_base: Context.ServiceClass<KnowledgeEmbedder, "@yolk-sdk/knowledge/KnowledgeEmbedder", KnowledgeEmbedderApi>;
|
|
11
|
-
declare class KnowledgeEmbedder extends KnowledgeEmbedder_base {}
|
|
12
|
-
|
|
13
|
-
export { KnowledgeEmbeddingError as EmbedderError, KnowledgeEmbedder, KnowledgeEmbedderApi, KnowledgeEmbedding };
|
|
9
|
+
export declare class KnowledgeEmbedder extends KnowledgeEmbedder_base {}
|
|
10
|
+
export { KnowledgeEmbeddingError as EmbedderError };
|
|
14
11
|
//# sourceMappingURL=embeddings.d.mts.map
|