@yolk-sdk/knowledge 0.1.0-canary.77 → 0.1.0-canary.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -8
- package/dist/agent.d.mts.map +1 -1
- package/dist/agent.mjs +2 -3
- package/dist/agent.mjs.map +1 -1
- package/dist/chunking.d.mts.map +1 -1
- package/dist/chunking.mjs.map +1 -1
- package/dist/context.mjs.map +1 -1
- package/dist/documents.d.mts +28 -19
- package/dist/documents.d.mts.map +1 -1
- package/dist/documents.mjs +14 -22
- package/dist/documents.mjs.map +1 -1
- package/dist/errors.mjs +3 -3
- package/dist/errors.mjs.map +1 -1
- package/dist/ingestion.d.mts +8 -8
- package/dist/ingestion.d.mts.map +1 -1
- package/dist/ingestion.mjs.map +1 -1
- package/dist/search.d.mts +1 -1
- package/dist/search.d.mts.map +1 -1
- package/dist/search.mjs +5 -7
- package/dist/search.mjs.map +1 -1
- package/package.json +3 -3
- package/src/agent.ts +17 -5
- package/src/chunking.ts +9 -0
- package/src/context.ts +1 -0
- package/src/documents.ts +44 -21
- package/src/errors.ts +3 -3
- package/src/index.ts +3 -0
- package/src/ingestion.ts +2 -0
- package/src/search.ts +14 -9
package/README.md
CHANGED
|
@@ -5,11 +5,11 @@ Domain-free knowledge document/source/file/context, ingestion, hybrid search, an
|
|
|
5
5
|
## Install
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
pnpm add @yolk-sdk/knowledge@canary @yolk-sdk/agent@canary effect@4.0.0-
|
|
8
|
+
pnpm add @yolk-sdk/knowledge@canary @yolk-sdk/agent@canary effect@4.0.0-rc.115
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
Canary APIs are unstable. Keep all `@yolk-sdk/*` packages on the same version.
|
|
12
|
-
Use the SDK's matching Effect version (`4.0.0-
|
|
12
|
+
Use the SDK's matching Effect version (`4.0.0-rc.115`) in host code.
|
|
13
13
|
Published package metadata requires Node.js 22+.
|
|
14
14
|
|
|
15
15
|
## Model
|
|
@@ -40,11 +40,12 @@ Optional files and chunks are modeled as `KnowledgeFile` and `KnowledgeChunk`. F
|
|
|
40
40
|
metadata and host-owned blob keys; `KnowledgeFileBlobStore` stores the bytes separately. Chunks are
|
|
41
41
|
search-index records. Hosts own blob storage, chunk persistence, transactions, and lifecycle.
|
|
42
42
|
|
|
43
|
-
`KnowledgeSource` is a separate ingestion ADT
|
|
43
|
+
`KnowledgeSource` is a separate ingestion ADT. Build variants with `Schema.TaggedStruct` constructors
|
|
44
|
+
from `@yolk-sdk/knowledge/documents`:
|
|
44
45
|
|
|
45
|
-
- `
|
|
46
|
-
- `
|
|
47
|
-
- `
|
|
46
|
+
- `KnowledgeFileSource.make({ ref, name?, mediaType? })` — host-owned `ref` plus optional name/media type
|
|
47
|
+
- `KnowledgeUrlSource.make({ url })` — host-owned URL string
|
|
48
|
+
- `KnowledgeTextSource.make({ label? })` — optional label; bytes/text arrive through `LoadedKnowledgeSource.content`
|
|
48
49
|
|
|
49
50
|
`IndexedKnowledgeDocument` is the search-index record. It carries `source`, indexing `status`, and
|
|
50
51
|
optional title/summary/error/hash/count metadata. It is distinct from the content-oriented
|
|
@@ -65,7 +66,9 @@ ready/pinned documents and exclude processing, error, or archived records as app
|
|
|
65
66
|
|
|
66
67
|
`KnowledgeScope` and `KnowledgeSearchScope` are opaque, caller-provided routing metadata. Hosts
|
|
67
68
|
construct and enforce them; the package does not infer identity, tenancy, or permissions from scope
|
|
68
|
-
ids or kinds.
|
|
69
|
+
ids or kinds. The **value** `KnowledgeSearchScope` is the single-id constructor
|
|
70
|
+
(`KnowledgeSearchScope.make({ id })`, tag `KnowledgeScope`). `KnowledgeSearchScopes.make({ ids })`
|
|
71
|
+
is the non-empty-ids variant. The **type** `KnowledgeSearchScope` remains the union of both.
|
|
69
72
|
|
|
70
73
|
## Subpaths
|
|
71
74
|
|
|
@@ -89,7 +92,14 @@ ids or kinds.
|
|
|
89
92
|
|
|
90
93
|
```ts
|
|
91
94
|
import { buildKnowledgeContext, KnowledgeStore } from '@yolk-sdk/knowledge'
|
|
92
|
-
import
|
|
95
|
+
import {
|
|
96
|
+
KnowledgeFileSource,
|
|
97
|
+
KnowledgeSearchScope,
|
|
98
|
+
KnowledgeSearchScopes,
|
|
99
|
+
KnowledgeTextSource,
|
|
100
|
+
KnowledgeUrlSource,
|
|
101
|
+
type KnowledgeDocument
|
|
102
|
+
} from '@yolk-sdk/knowledge/documents'
|
|
93
103
|
import { KnowledgeFileBlobStore } from '@yolk-sdk/knowledge/files'
|
|
94
104
|
import { makeKnowledgeLookupTool, makeKnowledgeManageTool } from '@yolk-sdk/knowledge/agent'
|
|
95
105
|
```
|
package/dist/agent.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.mts","names":[],"sources":["../src/agent.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"agent.d.mts","names":[],"sources":["../src/agent.ts"],"mappings":";;;;;;KA0EY,eAAA;EAAA,SACD,IAAA;EAAA,SACA,KAAK;AAAA;AAAA,KAGJ,qBAAA;EAAA,SACD,QAAA,EAAU,iBAAA;EAAA,SACV,KAAA;EAAA,SACA,OAAA,GAAU,aAAa;IAAA,SAAY,OAAA;EAAA;AAAA;AAAA,KAGlC,sBAAA;EAAA,SACD,EAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA;AAAA;AAAA,KAGC,uBAAA;EAAA,SACD,MAAA,GAAS,KAAA;IAAA,SACP,OAAA,EAAS,OAAA;IAAA,SACT,KAAA;IAAA,SACA,KAAA;IAAA,SACA,QAAA;IAAA,SACA,aAAA;EAAA,MACL,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,qBAAA,GAAwB,SAAA;EAAA,SACjD,GAAA,GAAM,KAAA;IAAA,SACJ,OAAA,EAAS,OAAA;IAAA,SACT,EAAA;IAAA,SACA,MAAA,GAAS,eAAA;EAAA,MACd,MAAA,CAAO,MAAA,CAAO,iBAAA,EAAmB,SAAA;AAAA;AAAA,KAG7B,uBAAA;EAAA,SACD,MAAA,GAAS,KAAA;IAAA,SACP,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;IAAA,SACR,KAAA;IAAA,SACA,OAAA;IAAA,SACA,MAAA;IAAA,SACA,OAAA;IAAA,SACA,YAAA,EAAc,qBAAA;EAAA,MACnB,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;EAAA,SACnC,eAAA,GAAkB,KAAA;IAAA,SAChB,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;IAAA,SACR,YAAA,EAAc,qBAAA;EAAA,MACnB,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;EAAA,SACnC,UAAA,GAAa,KAAA;IAAA,SACX,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;IAAA,SACR,QAAA;EAAA,MACL,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;EAAA,SACnC,MAAA,GAAS,KAAA;IAAA,SACP,OAAA,EAAS,OAAA;IAAA,SACT,MAAA,EAAQ,eAAA;EAAA,MACb,MAAA,CAAO,MAAA,CAAO,sBAAA,EAAwB,SAAA;AAAA;AAAA,cAoDjC,uBAAA,YACX,QAAA,EAAU,uBAAA,CAAwB,OAAA,GAClC,OAAA;EAAA,SAAoB,IAAA;EAAA,SAAwB,WAAA;AAAA,MAC3C,gBAAA,CAAiB,OAAA;AAAA,cAqCP,uBAAA,YACX,QAAA,EAAU,uBAAA,CAAwB,OAAA,GAClC,OAAA;EAAA,SAAoB,IAAA;EAAA,SAAwB,WAAA;AAAA,MAC3C,gBAAA,CAAiB,OAAA"}
|
package/dist/agent.mjs
CHANGED
|
@@ -49,7 +49,6 @@ const KnowledgeManageParams = Schema.Union([
|
|
|
49
49
|
target: KnowledgeTargetParamsSchema
|
|
50
50
|
})
|
|
51
51
|
]);
|
|
52
|
-
const unknownToMessage = (error) => error instanceof Error ? error.message : String(error);
|
|
53
52
|
const optionalValue = (value) => value ?? void 0;
|
|
54
53
|
const targetFromParams = (target) => target.scope === null || target.scope === void 0 ? { slug: target.slug } : {
|
|
55
54
|
slug: target.slug,
|
|
@@ -90,7 +89,7 @@ const makeKnowledgeLookupTool = (handlers, options = {}) => {
|
|
|
90
89
|
description: options.description ?? "Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.",
|
|
91
90
|
parameters: KnowledgeLookupParams,
|
|
92
91
|
access: "read",
|
|
93
|
-
invalidParamsMessage: (error) => `Invalid knowledge lookup arguments: ${
|
|
92
|
+
invalidParamsMessage: (error) => `Invalid knowledge lookup arguments: ${error instanceof Error ? error.message : String(error)}`,
|
|
94
93
|
execute: ({ call, context, params }) => Effect.gen(function* () {
|
|
95
94
|
if (params.operation === "search") {
|
|
96
95
|
const results = yield* handlers.search({
|
|
@@ -123,7 +122,7 @@ const makeKnowledgeManageTool = (handlers, options = {}) => {
|
|
|
123
122
|
description: options.description ?? "Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.",
|
|
124
123
|
parameters: KnowledgeManageParams,
|
|
125
124
|
access: "write",
|
|
126
|
-
invalidParamsMessage: (error) => `Invalid knowledge manage arguments: ${
|
|
125
|
+
invalidParamsMessage: (error) => `Invalid knowledge manage arguments: ${error instanceof Error ? error.message : String(error)}`,
|
|
127
126
|
execute: ({ call, context, params }) => Effect.gen(function* () {
|
|
128
127
|
switch (params.operation) {
|
|
129
128
|
case "upsert": {
|
package/dist/agent.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.mjs","names":[],"sources":["../src/agent.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport * as Schema from 'effect/Schema'\nimport { ToolError } from '@yolk-sdk/agent/loop'\nimport { ToolResult } from '@yolk-sdk/agent/protocol'\nimport { makeTool, type ToolRegistration } from '@yolk-sdk/agent/tools'\nimport {\n KnowledgeAvailabilitySchema,\n NonNegativeInteger,\n NonEmptyTrimmedString,\n PositiveInteger,\n type KnowledgeAvailability,\n type KnowledgeDocument\n} from './documents.ts'\n\nconst OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString))\nconst OptionalPositiveInteger = Schema.optional(Schema.NullOr(PositiveInteger))\nconst OptionalNonNegativeInteger = Schema.optional(Schema.NullOr(NonNegativeInteger))\nconst OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number))\n\nconst KnowledgeTargetParamsSchema = Schema.Struct({\n slug: NonEmptyTrimmedString,\n scope: OptionalNonEmptyTrimmedString\n})\n\nconst KnowledgeLookupParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n query: NonEmptyTrimmedString,\n limit: OptionalPositiveInteger,\n minScore: OptionalNumber,\n contextChunks: OptionalNonNegativeInteger\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n id: OptionalNonEmptyTrimmedString,\n target: Schema.optional(Schema.NullOr(KnowledgeTargetParamsSchema))\n })\n])\n\nconst KnowledgeManageParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('upsert'),\n target: KnowledgeTargetParamsSchema,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('set_availability'),\n target: KnowledgeTargetParamsSchema,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('rename_slug'),\n target: KnowledgeTargetParamsSchema,\n nextSlug: NonEmptyTrimmedString\n }),\n Schema.Struct({\n operation: Schema.Literal('delete'),\n target: KnowledgeTargetParamsSchema\n })\n])\n\ntype KnowledgeTargetParams = typeof KnowledgeTargetParamsSchema.Type\ntype KnowledgeLookupParams = typeof KnowledgeLookupParams.Type\ntype KnowledgeManageParams = typeof KnowledgeManageParams.Type\n\nexport type KnowledgeTarget = {\n readonly slug: string\n readonly scope?: string\n}\n\nexport type KnowledgeLookupResult = {\n readonly document: KnowledgeDocument\n readonly score?: number\n readonly context?: ReadonlyArray<{ readonly content: string }>\n}\n\nexport type KnowledgeSavedDocument = {\n readonly id: string\n readonly slug: string\n readonly title: string\n}\n\nexport type KnowledgeLookupHandlers<Context> = {\n readonly search: (input: {\n readonly context: Context\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n }) => Effect.Effect<ReadonlyArray<KnowledgeLookupResult>, ToolError>\n readonly get: (input: {\n readonly context: Context\n readonly id?: string\n readonly target?: KnowledgeTarget\n }) => Effect.Effect<KnowledgeDocument, ToolError>\n}\n\nexport type KnowledgeManageHandlers<Context> = {\n readonly upsert: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly title: string\n readonly purpose: string\n readonly origin: string\n readonly content: string\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly setAvailability: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly renameSlug: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly nextSlug: string\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly delete: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n}\n\nconst unknownToMessage = (error: unknown) =>\n error instanceof Error ? error.message : String(error)\n\nconst optionalValue = <Value>(value: Value | null | undefined) => value ?? undefined\n\nconst targetFromParams = (target: KnowledgeTargetParams): KnowledgeTarget =>\n target.scope === null || target.scope === undefined\n ? { slug: target.slug }\n : { slug: target.slug, scope: target.scope }\n\nconst optionalTargetFromParams = (target: KnowledgeTargetParams | null | undefined) =>\n target === null || target === undefined ? undefined : targetFromParams(target)\n\nconst formatDocument = (document: KnowledgeDocument) =>\n [\n `# ${document.title}`,\n `document_id: ${document.id}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `availability: ${document.availability}`,\n `status: ${document.status}`,\n '',\n document.content\n ].join('\\n')\n\nconst formatSearchResults = (results: ReadonlyArray<KnowledgeLookupResult>) =>\n results.length === 0\n ? 'No knowledge matches found.'\n : results\n .map(result =>\n [\n `## ${result.document.title}`,\n `document_id: ${result.document.id}`,\n `slug: ${result.document.slug}`,\n `purpose: ${result.document.purpose}`,\n `origin: ${result.document.origin}`,\n `availability: ${result.document.availability}`,\n result.score === undefined ? undefined : `score: ${result.score}`,\n '',\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.document.content\n ]\n .filter(line => line !== undefined)\n .join('\\n')\n )\n .join('\\n\\n')\n\nconst formatSaved = (verb: string, document: KnowledgeSavedDocument) =>\n [verb, `title: ${document.title}`, `slug: ${document.slug}`, `document_id: ${document.id}`].join(\n '\\n'\n )\n\nexport const makeKnowledgeLookupTool = <Context>(\n handlers: KnowledgeLookupHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_lookup'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.',\n parameters: KnowledgeLookupParams,\n access: 'read',\n invalidParamsMessage: error => `Invalid knowledge lookup arguments: ${unknownToMessage(error)}`,\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n if (params.operation === 'search') {\n const results = yield* handlers.search({\n context,\n query: params.query,\n limit: optionalValue(params.limit),\n minScore: optionalValue(params.minScore),\n contextChunks: optionalValue(params.contextChunks)\n })\n\n return ToolResult.make({ toolCallId: call.id, content: formatSearchResults(results) })\n }\n\n const document = yield* handlers.get({\n context,\n id: optionalValue(params.id),\n target: optionalTargetFromParams(params.target)\n })\n return ToolResult.make({ toolCallId: call.id, content: formatDocument(document) })\n })\n })\n}\n\nexport const makeKnowledgeManageTool = <Context>(\n handlers: KnowledgeManageHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_manage'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.',\n parameters: KnowledgeManageParams,\n access: 'write',\n invalidParamsMessage: error => `Invalid knowledge manage arguments: ${unknownToMessage(error)}`,\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n switch (params.operation) {\n case 'upsert': {\n const document = yield* handlers.upsert({\n context,\n target: targetFromParams(params.target),\n title: params.title,\n purpose: params.purpose,\n origin: params.origin,\n content: params.content,\n availability: params.availability\n })\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge upserted', document)\n })\n }\n case 'set_availability': {\n const document = yield* handlers.setAvailability({\n context,\n target: targetFromParams(params.target),\n availability: params.availability\n })\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge availability updated', document)\n })\n }\n case 'rename_slug': {\n const document = yield* handlers.renameSlug({\n context,\n target: targetFromParams(params.target),\n nextSlug: params.nextSlug\n })\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge slug renamed', document)\n })\n }\n case 'delete': {\n const document = yield* handlers.delete({\n context,\n target: targetFromParams(params.target)\n })\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge deleted', document)\n })\n }\n }\n\n return yield* Effect.fail(\n new ToolError({\n tool: call.name,\n message: 'Unsupported knowledge operation',\n cause: 'validation'\n })\n )\n })\n })\n}\n"],"mappings":";;;;;;;AAcA,MAAM,gCAAgC,OAAO,SAAS,OAAO,OAAO,qBAAqB,CAAC;AAC1F,MAAM,0BAA0B,OAAO,SAAS,OAAO,OAAO,eAAe,CAAC;AAC9E,MAAM,6BAA6B,OAAO,SAAS,OAAO,OAAO,kBAAkB,CAAC;AACpF,MAAM,iBAAiB,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC;AAEnE,MAAM,8BAA8B,OAAO,OAAO;CAChD,MAAM;CACN,OAAO;AACT,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM,CACzC,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,OAAO;CACP,OAAO;CACP,UAAU;CACV,eAAe;AACjB,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,IAAI;CACJ,QAAQ,OAAO,SAAS,OAAO,OAAO,2BAA2B,CAAC;AACpE,CAAC,CACH,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM;CACzC,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;EACR,OAAO;EACP,SAAS;EACT,QAAQ;EACR,SAAS;EACT,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,kBAAkB;EAC5C,QAAQ;EACR,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,aAAa;EACvC,QAAQ;EACR,UAAU;CACZ,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;CACV,CAAC;AACH,CAAC;AAgED,MAAM,oBAAoB,UACxB,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAEvD,MAAM,iBAAwB,UAAoC,SAAS,KAAA;AAE3E,MAAM,oBAAoB,WACxB,OAAO,UAAU,QAAQ,OAAO,UAAU,KAAA,IACtC,EAAE,MAAM,OAAO,KAAK,IACpB;CAAE,MAAM,OAAO;CAAM,OAAO,OAAO;AAAM;AAE/C,MAAM,4BAA4B,WAChC,WAAW,QAAQ,WAAW,KAAA,IAAY,KAAA,IAAY,iBAAiB,MAAM;AAE/E,MAAM,kBAAkB,aACtB;CACE,KAAK,SAAS;CACd,gBAAgB,SAAS;CACzB,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,WAAW,SAAS;CACpB;CACA,SAAS;AACX,EAAE,KAAK,IAAI;AAEb,MAAM,uBAAuB,YAC3B,QAAQ,WAAW,IACf,gCACA,QACG,KAAI,WACH;CACE,MAAM,OAAO,SAAS;CACtB,gBAAgB,OAAO,SAAS;CAChC,SAAS,OAAO,SAAS;CACzB,YAAY,OAAO,SAAS;CAC5B,WAAW,OAAO,SAAS;CAC3B,iBAAiB,OAAO,SAAS;CACjC,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,UAAU,OAAO;CAC1D;CACA,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,SAAS;AAC9E,EACG,QAAO,SAAQ,SAAS,KAAA,CAAS,EACjC,KAAK,IAAI,CACd,EACC,KAAK,MAAM;AAEpB,MAAM,eAAe,MAAc,aACjC;CAAC;CAAM,UAAU,SAAS;CAAS,SAAS,SAAS;CAAQ,gBAAgB,SAAS;AAAI,EAAE,KAC1F,IACF;AAEF,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,uBAAsB,UAAS,uCAAuC,iBAAiB,KAAK;EAC5F,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,IAAI,OAAO,cAAc,UAAU;IACjC,MAAM,UAAU,OAAO,SAAS,OAAO;KACrC;KACA,OAAO,OAAO;KACd,OAAO,cAAc,OAAO,KAAK;KACjC,UAAU,cAAc,OAAO,QAAQ;KACvC,eAAe,cAAc,OAAO,aAAa;IACnD,CAAC;IAED,OAAO,WAAW,KAAK;KAAE,YAAY,KAAK;KAAI,SAAS,oBAAoB,OAAO;IAAE,CAAC;GACvF;GAEA,MAAM,WAAW,OAAO,SAAS,IAAI;IACnC;IACA,IAAI,cAAc,OAAO,EAAE;IAC3B,QAAQ,yBAAyB,OAAO,MAAM;GAChD,CAAC;GACD,OAAO,WAAW,KAAK;IAAE,YAAY,KAAK;IAAI,SAAS,eAAe,QAAQ;GAAE,CAAC;EACnF,CAAC;CACL,CAAC;AACH;AAEA,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,uBAAsB,UAAS,uCAAuC,iBAAiB,KAAK;EAC5F,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,QAAQ,OAAO,WAAf;IACE,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,OAAO,OAAO;MACd,SAAS,OAAO;MAChB,QAAQ,OAAO;MACf,SAAS,OAAO;MAChB,cAAc,OAAO;KACvB,CAAC;KACD,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,sBAAsB,QAAQ;KACrD,CAAC;IACH;IACA,KAAK,oBAAoB;KACvB,MAAM,WAAW,OAAO,SAAS,gBAAgB;MAC/C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,cAAc,OAAO;KACvB,CAAC;KACD,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,kCAAkC,QAAQ;KACjE,CAAC;IACH;IACA,KAAK,eAAe;KAClB,MAAM,WAAW,OAAO,SAAS,WAAW;MAC1C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,UAAU,OAAO;KACnB,CAAC;KACD,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,0BAA0B,QAAQ;KACzD,CAAC;IACH;IACA,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;KACxC,CAAC;KACD,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,qBAAqB,QAAQ;KACpD,CAAC;IACH;GACF;GAEA,OAAO,OAAO,OAAO,KACnB,IAAI,UAAU;IACZ,MAAM,KAAK;IACX,SAAS;IACT,OAAO;GACT,CAAC,CACH;EACF,CAAC;CACL,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"agent.mjs","names":[],"sources":["../src/agent.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport * as Schema from 'effect/Schema'\nimport { ToolError } from '@yolk-sdk/agent/loop'\nimport { ToolResult } from '@yolk-sdk/agent/protocol'\nimport { makeTool, type ToolRegistration } from '@yolk-sdk/agent/tools'\nimport {\n KnowledgeAvailabilitySchema,\n NonNegativeInteger,\n NonEmptyTrimmedString,\n PositiveInteger,\n type KnowledgeAvailability,\n type KnowledgeDocument\n} from './documents.ts'\n\nconst OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString))\n\nconst OptionalPositiveInteger = Schema.optional(Schema.NullOr(PositiveInteger))\n\nconst OptionalNonNegativeInteger = Schema.optional(Schema.NullOr(NonNegativeInteger))\n\nconst OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number))\n\nconst KnowledgeTargetParamsSchema = Schema.Struct({\n slug: NonEmptyTrimmedString,\n scope: OptionalNonEmptyTrimmedString\n})\n\nconst KnowledgeLookupParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('search'),\n query: NonEmptyTrimmedString,\n limit: OptionalPositiveInteger,\n minScore: OptionalNumber,\n contextChunks: OptionalNonNegativeInteger\n }),\n Schema.Struct({\n operation: Schema.Literal('get'),\n id: OptionalNonEmptyTrimmedString,\n target: Schema.optional(Schema.NullOr(KnowledgeTargetParamsSchema))\n })\n])\n\nconst KnowledgeManageParams = Schema.Union([\n Schema.Struct({\n operation: Schema.Literal('upsert'),\n target: KnowledgeTargetParamsSchema,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('set_availability'),\n target: KnowledgeTargetParamsSchema,\n availability: KnowledgeAvailabilitySchema\n }),\n Schema.Struct({\n operation: Schema.Literal('rename_slug'),\n target: KnowledgeTargetParamsSchema,\n nextSlug: NonEmptyTrimmedString\n }),\n Schema.Struct({\n operation: Schema.Literal('delete'),\n target: KnowledgeTargetParamsSchema\n })\n])\n\ntype KnowledgeTargetParams = typeof KnowledgeTargetParamsSchema.Type\n\ntype KnowledgeLookupParams = typeof KnowledgeLookupParams.Type\n\ntype KnowledgeManageParams = typeof KnowledgeManageParams.Type\n\nexport type KnowledgeTarget = {\n readonly slug: string\n readonly scope?: string\n}\n\nexport type KnowledgeLookupResult = {\n readonly document: KnowledgeDocument\n readonly score?: number\n readonly context?: ReadonlyArray<{ readonly content: string }>\n}\n\nexport type KnowledgeSavedDocument = {\n readonly id: string\n readonly slug: string\n readonly title: string\n}\n\nexport type KnowledgeLookupHandlers<Context> = {\n readonly search: (input: {\n readonly context: Context\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n }) => Effect.Effect<ReadonlyArray<KnowledgeLookupResult>, ToolError>\n readonly get: (input: {\n readonly context: Context\n readonly id?: string\n readonly target?: KnowledgeTarget\n }) => Effect.Effect<KnowledgeDocument, ToolError>\n}\n\nexport type KnowledgeManageHandlers<Context> = {\n readonly upsert: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly title: string\n readonly purpose: string\n readonly origin: string\n readonly content: string\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly setAvailability: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly availability: KnowledgeAvailability\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly renameSlug: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n readonly nextSlug: string\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n readonly delete: (input: {\n readonly context: Context\n readonly target: KnowledgeTarget\n }) => Effect.Effect<KnowledgeSavedDocument, ToolError>\n}\n\nconst optionalValue = <Value>(value: Value | null | undefined) => value ?? undefined\n\nconst targetFromParams = (target: KnowledgeTargetParams): KnowledgeTarget =>\n target.scope === null || target.scope === undefined\n ? { slug: target.slug }\n : { slug: target.slug, scope: target.scope }\n\nconst optionalTargetFromParams = (target: KnowledgeTargetParams | null | undefined) =>\n target === null || target === undefined ? undefined : targetFromParams(target)\n\nconst formatDocument = (document: KnowledgeDocument) =>\n [\n `# ${document.title}`,\n `document_id: ${document.id}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `availability: ${document.availability}`,\n `status: ${document.status}`,\n '',\n document.content\n ].join('\\n')\n\nconst formatSearchResults = (results: ReadonlyArray<KnowledgeLookupResult>) =>\n results.length === 0\n ? 'No knowledge matches found.'\n : results\n .map(result =>\n [\n `## ${result.document.title}`,\n `document_id: ${result.document.id}`,\n `slug: ${result.document.slug}`,\n `purpose: ${result.document.purpose}`,\n `origin: ${result.document.origin}`,\n `availability: ${result.document.availability}`,\n result.score === undefined ? undefined : `score: ${result.score}`,\n '',\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.document.content\n ]\n .filter(line => line !== undefined)\n .join('\\n')\n )\n .join('\\n\\n')\n\nconst formatSaved = (verb: string, document: KnowledgeSavedDocument) =>\n [verb, `title: ${document.title}`, `slug: ${document.slug}`, `document_id: ${document.id}`].join(\n '\\n'\n )\n\nexport const makeKnowledgeLookupTool = <Context>(\n handlers: KnowledgeLookupHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_lookup'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.',\n parameters: KnowledgeLookupParams,\n access: 'read',\n invalidParamsMessage: error =>\n `Invalid knowledge lookup arguments: ${error instanceof Error ? error.message : String(error)}`,\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n if (params.operation === 'search') {\n const results = yield* handlers.search({\n context,\n query: params.query,\n limit: optionalValue(params.limit),\n minScore: optionalValue(params.minScore),\n contextChunks: optionalValue(params.contextChunks)\n })\n\n return ToolResult.make({ toolCallId: call.id, content: formatSearchResults(results) })\n }\n\n const document = yield* handlers.get({\n context,\n id: optionalValue(params.id),\n target: optionalTargetFromParams(params.target)\n })\n\n return ToolResult.make({ toolCallId: call.id, content: formatDocument(document) })\n })\n })\n}\n\nexport const makeKnowledgeManageTool = <Context>(\n handlers: KnowledgeManageHandlers<Context>,\n options: { readonly name?: string; readonly description?: string } = {}\n): ToolRegistration<Context> => {\n const name = options.name ?? 'knowledge_manage'\n\n return makeTool({\n name,\n description:\n options.description ??\n 'Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.',\n parameters: KnowledgeManageParams,\n access: 'write',\n invalidParamsMessage: error =>\n `Invalid knowledge manage arguments: ${error instanceof Error ? error.message : String(error)}`,\n execute: ({ call, context, params }) =>\n Effect.gen(function* () {\n switch (params.operation) {\n case 'upsert': {\n const document = yield* handlers.upsert({\n context,\n target: targetFromParams(params.target),\n title: params.title,\n purpose: params.purpose,\n origin: params.origin,\n content: params.content,\n availability: params.availability\n })\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge upserted', document)\n })\n }\n\n case 'set_availability': {\n const document = yield* handlers.setAvailability({\n context,\n target: targetFromParams(params.target),\n availability: params.availability\n })\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge availability updated', document)\n })\n }\n\n case 'rename_slug': {\n const document = yield* handlers.renameSlug({\n context,\n target: targetFromParams(params.target),\n nextSlug: params.nextSlug\n })\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge slug renamed', document)\n })\n }\n\n case 'delete': {\n const document = yield* handlers.delete({\n context,\n target: targetFromParams(params.target)\n })\n\n return ToolResult.make({\n toolCallId: call.id,\n content: formatSaved('Knowledge deleted', document)\n })\n }\n }\n\n return yield* Effect.fail(\n new ToolError({\n tool: call.name,\n message: 'Unsupported knowledge operation',\n cause: 'validation'\n })\n )\n })\n })\n}\n"],"mappings":";;;;;;;AAcA,MAAM,gCAAgC,OAAO,SAAS,OAAO,OAAO,qBAAqB,CAAC;AAE1F,MAAM,0BAA0B,OAAO,SAAS,OAAO,OAAO,eAAe,CAAC;AAE9E,MAAM,6BAA6B,OAAO,SAAS,OAAO,OAAO,kBAAkB,CAAC;AAEpF,MAAM,iBAAiB,OAAO,SAAS,OAAO,OAAO,OAAO,MAAM,CAAC;AAEnE,MAAM,8BAA8B,OAAO,OAAO;CAChD,MAAM;CACN,OAAO;AACT,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM,CACzC,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,QAAQ;CAClC,OAAO;CACP,OAAO;CACP,UAAU;CACV,eAAe;AACjB,CAAC,GACD,OAAO,OAAO;CACZ,WAAW,OAAO,QAAQ,KAAK;CAC/B,IAAI;CACJ,QAAQ,OAAO,SAAS,OAAO,OAAO,2BAA2B,CAAC;AACpE,CAAC,CACH,CAAC;AAED,MAAM,wBAAwB,OAAO,MAAM;CACzC,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;EACR,OAAO;EACP,SAAS;EACT,QAAQ;EACR,SAAS;EACT,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,kBAAkB;EAC5C,QAAQ;EACR,cAAc;CAChB,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,aAAa;EACvC,QAAQ;EACR,UAAU;CACZ,CAAC;CACD,OAAO,OAAO;EACZ,WAAW,OAAO,QAAQ,QAAQ;EAClC,QAAQ;CACV,CAAC;AACH,CAAC;AAkED,MAAM,iBAAwB,UAAoC,SAAS,KAAA;AAE3E,MAAM,oBAAoB,WACxB,OAAO,UAAU,QAAQ,OAAO,UAAU,KAAA,IACtC,EAAE,MAAM,OAAO,KAAK,IACpB;CAAE,MAAM,OAAO;CAAM,OAAO,OAAO;AAAM;AAE/C,MAAM,4BAA4B,WAChC,WAAW,QAAQ,WAAW,KAAA,IAAY,KAAA,IAAY,iBAAiB,MAAM;AAE/E,MAAM,kBAAkB,aACtB;CACE,KAAK,SAAS;CACd,gBAAgB,SAAS;CACzB,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,WAAW,SAAS;CACpB;CACA,SAAS;AACX,EAAE,KAAK,IAAI;AAEb,MAAM,uBAAuB,YAC3B,QAAQ,WAAW,IACf,gCACA,QACG,KAAI,WACH;CACE,MAAM,OAAO,SAAS;CACtB,gBAAgB,OAAO,SAAS;CAChC,SAAS,OAAO,SAAS;CACzB,YAAY,OAAO,SAAS;CAC5B,WAAW,OAAO,SAAS;CAC3B,iBAAiB,OAAO,SAAS;CACjC,OAAO,UAAU,KAAA,IAAY,KAAA,IAAY,UAAU,OAAO;CAC1D;CACA,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,SAAS;AAC9E,EACG,QAAO,SAAQ,SAAS,KAAA,CAAS,EACjC,KAAK,IAAI,CACd,EACC,KAAK,MAAM;AAEpB,MAAM,eAAe,MAAc,aACjC;CAAC;CAAM,UAAU,SAAS;CAAS,SAAS,SAAS;CAAQ,gBAAgB,SAAS;AAAI,EAAE,KAC1F,IACF;AAEF,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,uBAAsB,UACpB,uCAAuC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC9F,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,IAAI,OAAO,cAAc,UAAU;IACjC,MAAM,UAAU,OAAO,SAAS,OAAO;KACrC;KACA,OAAO,OAAO;KACd,OAAO,cAAc,OAAO,KAAK;KACjC,UAAU,cAAc,OAAO,QAAQ;KACvC,eAAe,cAAc,OAAO,aAAa;IACnD,CAAC;IAED,OAAO,WAAW,KAAK;KAAE,YAAY,KAAK;KAAI,SAAS,oBAAoB,OAAO;IAAE,CAAC;GACvF;GAEA,MAAM,WAAW,OAAO,SAAS,IAAI;IACnC;IACA,IAAI,cAAc,OAAO,EAAE;IAC3B,QAAQ,yBAAyB,OAAO,MAAM;GAChD,CAAC;GAED,OAAO,WAAW,KAAK;IAAE,YAAY,KAAK;IAAI,SAAS,eAAe,QAAQ;GAAE,CAAC;EACnF,CAAC;CACL,CAAC;AACH;AAEA,MAAa,2BACX,UACA,UAAqE,CAAC,MACxC;CAG9B,OAAO,SAAS;EACd,MAHW,QAAQ,QAAQ;EAI3B,aACE,QAAQ,eACR;EACF,YAAY;EACZ,QAAQ;EACR,uBAAsB,UACpB,uCAAuC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EAC9F,UAAU,EAAE,MAAM,SAAS,aACzB,OAAO,IAAI,aAAa;GACtB,QAAQ,OAAO,WAAf;IACE,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,OAAO,OAAO;MACd,SAAS,OAAO;MAChB,QAAQ,OAAO;MACf,SAAS,OAAO;MAChB,cAAc,OAAO;KACvB,CAAC;KAED,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,sBAAsB,QAAQ;KACrD,CAAC;IACH;IAEA,KAAK,oBAAoB;KACvB,MAAM,WAAW,OAAO,SAAS,gBAAgB;MAC/C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,cAAc,OAAO;KACvB,CAAC;KAED,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,kCAAkC,QAAQ;KACjE,CAAC;IACH;IAEA,KAAK,eAAe;KAClB,MAAM,WAAW,OAAO,SAAS,WAAW;MAC1C;MACA,QAAQ,iBAAiB,OAAO,MAAM;MACtC,UAAU,OAAO;KACnB,CAAC;KAED,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,0BAA0B,QAAQ;KACzD,CAAC;IACH;IAEA,KAAK,UAAU;KACb,MAAM,WAAW,OAAO,SAAS,OAAO;MACtC;MACA,QAAQ,iBAAiB,OAAO,MAAM;KACxC,CAAC;KAED,OAAO,WAAW,KAAK;MACrB,YAAY,KAAK;MACjB,SAAS,YAAY,qBAAqB,QAAQ;KACpD,CAAC;IACH;GACF;GAEA,OAAO,OAAO,OAAO,KACnB,IAAI,UAAU;IACZ,MAAM,KAAK;IACX,SAAS;IACT,OAAO;GACT,CAAC,CACH;EACF,CAAC;CACL,CAAC;AACH"}
|
package/dist/chunking.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.d.mts","names":[],"sources":["../src/chunking.ts"],"mappings":";;;;;
|
|
1
|
+
{"version":3,"file":"chunking.d.mts","names":[],"sources":["../src/chunking.ts"],"mappings":";;;;;KAWY,2BAAA;EAAA,SACD,OAAA;EAAA,SACA,UAAA;EAAA,SACA,OAAA;EAAA,SACA,SAAA;EAAA,SACA,QAAA,GAAW,iBAAiB;AAAA;AAAA,KAG3B,mBAAA;EAAA,SACD,KAAA,GACP,KAAA,EAAO,2BAAA,KACJ,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,cAAA,GAAiB,sBAAA;AAAA;AAAA,cACnD,qBAAA;cAEY,gBAAA,SAAyB,qBAErC;AAAA,cAEY,yBAAA,GAA6B,IAAY;AAAA,cAqHzC,kBAAA,GAAsB,KAAA,EAAO,2BAAA,EAA6B,SAAA,aAAiB,MAAA,CAAA,MAAA;;;;;;;;;;;cAkC3E,2BAAA,GAA+B,KAAA;EAAA,SACjC,SAAA;AAAA,MACP,mBAEF;AAAA,cAEW,2BAAA,GACX,KAAA;EAAA,SAAkB,SAAA;AAAA,MAAwC,KAAA,CAAA,KAAA,CAAA,gBAAA"}
|
package/dist/chunking.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type { KnowledgeChunk, KnowledgeMetadata } from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly content: string\n readonly maxTokens?: number\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n readonly chunk: (\n input: ChunkKnowledgeDocumentInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n text\n .replace(/\\r\\n/g, '\\n')\n .replace(/[\\t ]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()\n\nconst splitSentences = (text: string) =>\n text\n .split(PARAGRAPH_BREAK_PATTERN)\n .flatMap(part => {\n if (part.length === 0) {\n return []\n }\n\n if (/^\\n{2,}$/.test(part)) {\n return ['\\n\\n']\n }\n\n return part.match(SENTENCE_PATTERN) ?? [part]\n })\n .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n const tokens = encode(text)\n return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n const words = text.match(WORD_PATTERN) ?? []\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n for (const word of words) {\n const wordTokenCount = countTokens(word)\n\n if (wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n chunks.push(...splitEncodedTokens(word, maxTokens))\n current = ''\n currentTokenCount = 0\n continue\n }\n\n if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n current = word\n currentTokenCount = wordTokenCount\n continue\n }\n\n current = `${current}${word}`\n currentTokenCount += wordTokenCount\n }\n\n const finalChunk = current.trim()\n if (finalChunk.length > 0) {\n chunks.push(finalChunk)\n }\n\n return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n units.flatMap(unit =>\n countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n const pushCurrent = () => {\n const content = current.trim()\n if (content.length > 0) {\n chunks.push(content)\n }\n }\n\n for (const unit of units) {\n const unitTokenCount = countTokens(unit)\n\n if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n pushCurrent()\n current = unit\n currentTokenCount = unitTokenCount\n continue\n }\n\n current = `${current}${unit}`\n currentTokenCount += unitTokenCount\n }\n\n pushCurrent()\n return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n Effect.gen(function* () {\n if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n )\n }\n\n const sanitized = sanitizeText(input.content)\n if (sanitized.length === 0) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Cannot chunk empty content' })\n )\n }\n\n const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n const contents = buildChunkContents(units, maxTokens)\n\n if (contents.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n }\n\n return contents.map((content, position) => ({\n id: `${input.documentId}:chunk:${position}`,\n scopeId: input.scopeId,\n documentId: input.documentId,\n content,\n position,\n tokenCount: countTokens(content),\n metadata: input.metadata\n })) satisfies ReadonlyArray<KnowledgeChunk>\n })\n\nexport const makeDefaultKnowledgeChunker = (input: {\n readonly maxTokens: number\n}): KnowledgeChunkerApi => ({\n chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (\n input: { readonly maxTokens: number } = { maxTokens: 512 }\n) => Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAKA,MAAM,mBAAmB;
|
|
1
|
+
{"version":3,"file":"chunking.mjs","names":[],"sources":["../src/chunking.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'\nimport type { KnowledgeChunk, KnowledgeMetadata } from './documents.ts'\nimport { KnowledgeChunkingError } from './errors.ts'\n\nconst SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:[\"')\\]]+)?\\s*|[^.!?]+$/g\n\nconst PARAGRAPH_BREAK_PATTERN = /(\\n{2,})/\n\nconst WORD_PATTERN = /\\S+\\s*/g\n\nexport type ChunkKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly content: string\n readonly maxTokens?: number\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeChunkerApi = {\n readonly chunk: (\n input: ChunkKnowledgeDocumentInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>\n}\n\nexport class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(\n '@yolk-sdk/knowledge/KnowledgeChunker'\n) {}\n\nexport const countKnowledgeChunkTokens = (text: string) => countTokens(text)\n\nconst sanitizeText = (text: string) =>\n text\n .replace(/\\r\\n/g, '\\n')\n .replace(/[\\t ]+/g, ' ')\n .replace(/\\n{3,}/g, '\\n\\n')\n .trim()\n\nconst splitSentences = (text: string) =>\n text\n .split(PARAGRAPH_BREAK_PATTERN)\n .flatMap(part => {\n if (part.length === 0) {\n return []\n }\n\n if (/^\\n{2,}$/.test(part)) {\n return ['\\n\\n']\n }\n\n return part.match(SENTENCE_PATTERN) ?? [part]\n })\n .filter(unit => unit.length > 0)\n\nconst splitEncodedTokens = (text: string, maxTokens: number) => {\n const tokens = encode(text)\n\n return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>\n decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()\n ).filter(chunk => chunk.length > 0)\n}\n\nconst splitOversizedUnit = (text: string, maxTokens: number) => {\n const words = text.match(WORD_PATTERN) ?? []\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n for (const word of words) {\n const wordTokenCount = countTokens(word)\n\n if (wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n chunks.push(...splitEncodedTokens(word, maxTokens))\n current = ''\n currentTokenCount = 0\n continue\n }\n\n if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {\n const currentChunk = current.trim()\n\n if (currentChunk.length > 0) {\n chunks.push(currentChunk)\n }\n\n current = word\n currentTokenCount = wordTokenCount\n continue\n }\n\n current = `${current}${word}`\n currentTokenCount += wordTokenCount\n }\n\n const finalChunk = current.trim()\n\n if (finalChunk.length > 0) {\n chunks.push(finalChunk)\n }\n\n return chunks\n}\n\nconst splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>\n units.flatMap(unit =>\n countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)\n )\n\nconst buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {\n const chunks: string[] = []\n let current = ''\n let currentTokenCount = 0\n\n const pushCurrent = () => {\n const content = current.trim()\n\n if (content.length > 0) {\n chunks.push(content)\n }\n }\n\n for (const unit of units) {\n const unitTokenCount = countTokens(unit)\n\n if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {\n pushCurrent()\n current = unit\n currentTokenCount = unitTokenCount\n continue\n }\n\n current = `${current}${unit}`\n currentTokenCount += unitTokenCount\n }\n\n pushCurrent()\n\n return chunks\n}\n\nexport const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>\n Effect.gen(function* () {\n if (!Number.isInteger(maxTokens) || maxTokens < 1) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })\n )\n }\n\n const sanitized = sanitizeText(input.content)\n\n if (sanitized.length === 0) {\n return yield* Effect.fail(\n new KnowledgeChunkingError({ message: 'Cannot chunk empty content' })\n )\n }\n\n const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)\n const contents = buildChunkContents(units, maxTokens)\n\n if (contents.length === 0) {\n return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))\n }\n\n return contents.map((content, position) => ({\n id: `${input.documentId}:chunk:${position}`,\n scopeId: input.scopeId,\n documentId: input.documentId,\n content,\n position,\n tokenCount: countTokens(content),\n metadata: input.metadata\n })) satisfies ReadonlyArray<KnowledgeChunk>\n })\n\nexport const makeDefaultKnowledgeChunker = (input: {\n readonly maxTokens: number\n}): KnowledgeChunkerApi => ({\n chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)\n})\n\nexport const DefaultKnowledgeChunkerLive = (\n input: { readonly maxTokens: number } = { maxTokens: 512 }\n) => Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))\n"],"mappings":";;;;AAKA,MAAM,mBAAmB;AAEzB,MAAM,0BAA0B;AAEhC,MAAM,eAAe;AAgBrB,IAAa,mBAAb,cAAsC,QAAQ,QAA+C,EAC3F,sCACF,EAAE,CAAC;AAEH,MAAa,6BAA6B,SAAiB,YAAY,IAAI;AAE3E,MAAM,gBAAgB,SACpB,KACG,QAAQ,SAAS,IAAI,EACrB,QAAQ,WAAW,GAAG,EACtB,QAAQ,WAAW,MAAM,EACzB,KAAK;AAEV,MAAM,kBAAkB,SACtB,KACG,MAAM,uBAAuB,EAC7B,SAAQ,SAAQ;CACf,IAAI,KAAK,WAAW,GAClB,OAAO,CAAC;CAGV,IAAI,WAAW,KAAK,IAAI,GACtB,OAAO,CAAC,MAAM;CAGhB,OAAO,KAAK,MAAM,gBAAgB,KAAK,CAAC,IAAI;AAC9C,CAAC,EACA,QAAO,SAAQ,KAAK,SAAS,CAAC;AAEnC,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,SAAS,OAAO,IAAI;CAE1B,OAAO,MAAM,KAAK,EAAE,QAAQ,KAAK,KAAK,OAAO,SAAS,SAAS,EAAE,IAAI,GAAG,UACtE,OAAO,OAAO,MAAM,QAAQ,WAAW,QAAQ,YAAY,SAAS,CAAC,EAAE,KAAK,CAC9E,EAAE,QAAO,UAAS,MAAM,SAAS,CAAC;AACpC;AAEA,MAAM,sBAAsB,MAAc,cAAsB;CAC9D,MAAM,QAAQ,KAAK,MAAM,YAAY,KAAK,CAAC;CAC3C,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,iBAAiB,WAAW;GAC9B,MAAM,eAAe,QAAQ,KAAK;GAElC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,OAAO,KAAK,GAAG,mBAAmB,MAAM,SAAS,CAAC;GAClD,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,MAAM,eAAe,QAAQ,KAAK;GAElC,IAAI,aAAa,SAAS,GACxB,OAAO,KAAK,YAAY;GAG1B,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,MAAM,aAAa,QAAQ,KAAK;CAEhC,IAAI,WAAW,SAAS,GACtB,OAAO,KAAK,UAAU;CAGxB,OAAO;AACT;AAEA,MAAM,oBAAoB,OAA8B,cACtD,MAAM,SAAQ,SACZ,YAAY,IAAI,KAAK,YAAY,CAAC,IAAI,IAAI,mBAAmB,MAAM,SAAS,CAC9E;AAEF,MAAM,sBAAsB,OAA8B,cAAsB;CAC9E,MAAM,SAAmB,CAAC;CAC1B,IAAI,UAAU;CACd,IAAI,oBAAoB;CAExB,MAAM,oBAAoB;EACxB,MAAM,UAAU,QAAQ,KAAK;EAE7B,IAAI,QAAQ,SAAS,GACnB,OAAO,KAAK,OAAO;CAEvB;CAEA,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,iBAAiB,YAAY,IAAI;EAEvC,IAAI,oBAAoB,KAAK,oBAAoB,iBAAiB,WAAW;GAC3E,YAAY;GACZ,UAAU;GACV,oBAAoB;GACpB;EACF;EAEA,UAAU,GAAG,UAAU;EACvB,qBAAqB;CACvB;CAEA,YAAY;CAEZ,OAAO;AACT;AAEA,MAAa,sBAAsB,OAAoC,cACrE,OAAO,IAAI,aAAa;CACtB,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6CAA6C,CAAC,CACtF;CAGF,MAAM,YAAY,aAAa,MAAM,OAAO;CAE5C,IAAI,UAAU,WAAW,GACvB,OAAO,OAAO,OAAO,KACnB,IAAI,uBAAuB,EAAE,SAAS,6BAA6B,CAAC,CACtE;CAIF,MAAM,WAAW,mBADH,iBAAiB,eAAe,SAAS,GAAG,SAClB,GAAG,SAAS;CAEpD,IAAI,SAAS,WAAW,GACtB,OAAO,OAAO,OAAO,KAAK,IAAI,uBAAuB,EAAE,SAAS,qBAAqB,CAAC,CAAC;CAGzF,OAAO,SAAS,KAAK,SAAS,cAAc;EAC1C,IAAI,GAAG,MAAM,WAAW,SAAS;EACjC,SAAS,MAAM;EACf,YAAY,MAAM;EAClB;EACA;EACA,YAAY,YAAY,OAAO;EAC/B,UAAU,MAAM;CAClB,EAAE;AACJ,CAAC;AAEH,MAAa,+BAA+B,WAEhB,EAC1B,QAAO,aAAY,mBAAmB,UAAU,SAAS,aAAa,MAAM,SAAS,EACvF;AAEA,MAAa,+BACX,QAAwC,EAAE,WAAW,IAAI,MACtD,MAAM,QAAQ,kBAAkB,4BAA4B,KAAK,CAAC"}
|
package/dist/context.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context.mjs","names":["Arr"],"sources":["../src/context.ts"],"sourcesContent":["import { Array as Arr } from 'effect'\nimport type { KnowledgeDocument } from './documents.ts'\n\nexport type BuildKnowledgeContextInput = {\n readonly documents: ReadonlyArray<KnowledgeDocument>\n readonly maxCharacters: number\n}\n\nconst truncationMarker = '\\n\\n[truncated: pinned knowledge exceeded context budget]'\n\nconst documentHeader = (document: KnowledgeDocument) =>\n [\n `## ${document.title}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `status: ${document.status}`,\n `availability: ${document.availability}`,\n document.reviewedAt === undefined ? undefined : `reviewed_at: ${document.reviewedAt.toString()}`\n ]\n .filter(line => line !== undefined && line.trim().length > 0)\n .join('\\n')\n\nconst documentBody = (document: KnowledgeDocument) =>\n [documentHeader(document), document.summary, document.content]\n .filter(section => section !== undefined && section.trim().length > 0)\n .join('\\n')\n\nexport const buildKnowledgeContext = (input: BuildKnowledgeContextInput) => {\n const sections = Arr.map(input.documents, documentBody).filter(\n section => section.trim().length > 0\n )\n const header = '# Pinned knowledge\\nUse this durable knowledge as high-priority context.'\n const body = sections.join('\\n\\n')\n const context = body.length === 0 ? '' : `${header}\\n\\n${body}`\n\n if (context.length <= input.maxCharacters) {\n return context\n }\n\n if (input.maxCharacters <= truncationMarker.length) {\n return context.slice(0, input.maxCharacters).trimEnd()\n }\n\n return `${context.slice(0, input.maxCharacters - truncationMarker.length).trimEnd()}${truncationMarker}`\n}\n"],"mappings":";;AAQA,MAAM,mBAAmB;AAEzB,MAAM,kBAAkB,aACtB;CACE,MAAM,SAAS;CACf,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,SAAS,eAAe,KAAA,IAAY,KAAA,IAAY,gBAAgB,SAAS,WAAW,SAAS;AAC/F,EACG,QAAO,SAAQ,SAAS,KAAA,KAAa,KAAK,KAAK,EAAE,SAAS,CAAC,EAC3D,KAAK,IAAI;AAEd,MAAM,gBAAgB,aACpB;CAAC,eAAe,QAAQ;CAAG,SAAS;CAAS,SAAS;AAAO,EAC1D,QAAO,YAAW,YAAY,KAAA,KAAa,QAAQ,KAAK,EAAE,SAAS,CAAC,EACpE,KAAK,IAAI;AAEd,MAAa,yBAAyB,UAAsC;CAC1E,MAAM,WAAWA,MAAI,IAAI,MAAM,WAAW,YAAY,EAAE,QACtD,YAAW,QAAQ,KAAK,EAAE,SAAS,CACrC;
|
|
1
|
+
{"version":3,"file":"context.mjs","names":["Arr"],"sources":["../src/context.ts"],"sourcesContent":["import { Array as Arr } from 'effect'\nimport type { KnowledgeDocument } from './documents.ts'\n\nexport type BuildKnowledgeContextInput = {\n readonly documents: ReadonlyArray<KnowledgeDocument>\n readonly maxCharacters: number\n}\n\nconst truncationMarker = '\\n\\n[truncated: pinned knowledge exceeded context budget]'\n\nconst documentHeader = (document: KnowledgeDocument) =>\n [\n `## ${document.title}`,\n `slug: ${document.slug}`,\n `purpose: ${document.purpose}`,\n `origin: ${document.origin}`,\n `status: ${document.status}`,\n `availability: ${document.availability}`,\n document.reviewedAt === undefined ? undefined : `reviewed_at: ${document.reviewedAt.toString()}`\n ]\n .filter(line => line !== undefined && line.trim().length > 0)\n .join('\\n')\n\nconst documentBody = (document: KnowledgeDocument) =>\n [documentHeader(document), document.summary, document.content]\n .filter(section => section !== undefined && section.trim().length > 0)\n .join('\\n')\n\nexport const buildKnowledgeContext = (input: BuildKnowledgeContextInput) => {\n const sections = Arr.map(input.documents, documentBody).filter(\n section => section.trim().length > 0\n )\n\n const header = '# Pinned knowledge\\nUse this durable knowledge as high-priority context.'\n const body = sections.join('\\n\\n')\n const context = body.length === 0 ? '' : `${header}\\n\\n${body}`\n\n if (context.length <= input.maxCharacters) {\n return context\n }\n\n if (input.maxCharacters <= truncationMarker.length) {\n return context.slice(0, input.maxCharacters).trimEnd()\n }\n\n return `${context.slice(0, input.maxCharacters - truncationMarker.length).trimEnd()}${truncationMarker}`\n}\n"],"mappings":";;AAQA,MAAM,mBAAmB;AAEzB,MAAM,kBAAkB,aACtB;CACE,MAAM,SAAS;CACf,SAAS,SAAS;CAClB,YAAY,SAAS;CACrB,WAAW,SAAS;CACpB,WAAW,SAAS;CACpB,iBAAiB,SAAS;CAC1B,SAAS,eAAe,KAAA,IAAY,KAAA,IAAY,gBAAgB,SAAS,WAAW,SAAS;AAC/F,EACG,QAAO,SAAQ,SAAS,KAAA,KAAa,KAAK,KAAK,EAAE,SAAS,CAAC,EAC3D,KAAK,IAAI;AAEd,MAAM,gBAAgB,aACpB;CAAC,eAAe,QAAQ;CAAG,SAAS;CAAS,SAAS;AAAO,EAC1D,QAAO,YAAW,YAAY,KAAA,KAAa,QAAQ,KAAK,EAAE,SAAS,CAAC,EACpE,KAAK,IAAI;AAEd,MAAa,yBAAyB,UAAsC;CAC1E,MAAM,WAAWA,MAAI,IAAI,MAAM,WAAW,YAAY,EAAE,QACtD,YAAW,QAAQ,KAAK,EAAE,SAAS,CACrC;CAEA,MAAM,SAAS;CACf,MAAM,OAAO,SAAS,KAAK,MAAM;CACjC,MAAM,UAAU,KAAK,WAAW,IAAI,KAAK,GAAG,OAAO,MAAM;CAEzD,IAAI,QAAQ,UAAU,MAAM,eAC1B,OAAO;CAGT,IAAI,MAAM,iBAAiB,IACzB,OAAO,QAAQ,MAAM,GAAG,MAAM,aAAa,EAAE,QAAQ;CAGvD,OAAO,GAAG,QAAQ,MAAM,GAAG,MAAM,gBAAgB,EAAuB,EAAE,QAAQ,IAAI;AACxF"}
|
package/dist/documents.d.mts
CHANGED
|
@@ -87,40 +87,49 @@ declare const KnowledgeChunkSchema: Schema.Struct<{
|
|
|
87
87
|
readonly metadata: Schema.optional<Schema.$Record<Schema.String, Schema.Unknown>>;
|
|
88
88
|
}>;
|
|
89
89
|
type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>;
|
|
90
|
-
declare const
|
|
91
|
-
readonly _tag: Schema.Literal<"KnowledgeScope">;
|
|
90
|
+
declare const KnowledgeSearchScope: Schema.TaggedStruct<"KnowledgeScope", {
|
|
92
91
|
readonly id: Schema.Trimmed;
|
|
93
|
-
}
|
|
94
|
-
|
|
92
|
+
}>;
|
|
93
|
+
declare const KnowledgeSearchScopes: Schema.TaggedStruct<"KnowledgeScopes", {
|
|
94
|
+
readonly ids: Schema.NonEmptyArray<Schema.Trimmed>;
|
|
95
|
+
}>;
|
|
96
|
+
declare const KnowledgeSearchScopeSchema: Schema.Union<readonly [Schema.TaggedStruct<"KnowledgeScope", {
|
|
97
|
+
readonly id: Schema.Trimmed;
|
|
98
|
+
}>, Schema.TaggedStruct<"KnowledgeScopes", {
|
|
95
99
|
readonly ids: Schema.NonEmptyArray<Schema.Trimmed>;
|
|
96
100
|
}>]>;
|
|
97
|
-
type KnowledgeSearchScope =
|
|
98
|
-
declare const
|
|
99
|
-
readonly
|
|
101
|
+
type KnowledgeSearchScope = typeof KnowledgeSearchScopeSchema.Type;
|
|
102
|
+
declare const KnowledgeFileSource: Schema.TaggedStruct<"File", {
|
|
103
|
+
readonly ref: Schema.Trimmed;
|
|
104
|
+
readonly name: Schema.optional<Schema.Trimmed>;
|
|
105
|
+
readonly mediaType: Schema.optional<Schema.Trimmed>;
|
|
106
|
+
}>;
|
|
107
|
+
declare const KnowledgeUrlSource: Schema.TaggedStruct<"Url", {
|
|
108
|
+
readonly url: Schema.Trimmed;
|
|
109
|
+
}>;
|
|
110
|
+
declare const KnowledgeTextSource: Schema.TaggedStruct<"Text", {
|
|
111
|
+
readonly label: Schema.optional<Schema.Trimmed>;
|
|
112
|
+
}>;
|
|
113
|
+
declare const KnowledgeSourceSchema: Schema.Union<readonly [Schema.TaggedStruct<"File", {
|
|
100
114
|
readonly ref: Schema.Trimmed;
|
|
101
115
|
readonly name: Schema.optional<Schema.Trimmed>;
|
|
102
116
|
readonly mediaType: Schema.optional<Schema.Trimmed>;
|
|
103
|
-
}>, Schema.
|
|
104
|
-
readonly _tag: Schema.Literal<"Url">;
|
|
117
|
+
}>, Schema.TaggedStruct<"Url", {
|
|
105
118
|
readonly url: Schema.Trimmed;
|
|
106
|
-
}>, Schema.
|
|
107
|
-
readonly _tag: Schema.Literal<"Text">;
|
|
119
|
+
}>, Schema.TaggedStruct<"Text", {
|
|
108
120
|
readonly label: Schema.optional<Schema.Trimmed>;
|
|
109
121
|
}>]>;
|
|
110
|
-
type KnowledgeSource =
|
|
122
|
+
type KnowledgeSource = typeof KnowledgeSourceSchema.Type;
|
|
111
123
|
declare const IndexedKnowledgeDocumentSchema: Schema.Struct<{
|
|
112
124
|
readonly id: Schema.Trimmed;
|
|
113
125
|
readonly scopeId: Schema.Trimmed;
|
|
114
|
-
readonly source: Schema.Union<readonly [Schema.
|
|
115
|
-
readonly _tag: Schema.Literal<"File">;
|
|
126
|
+
readonly source: Schema.Union<readonly [Schema.TaggedStruct<"File", {
|
|
116
127
|
readonly ref: Schema.Trimmed;
|
|
117
128
|
readonly name: Schema.optional<Schema.Trimmed>;
|
|
118
129
|
readonly mediaType: Schema.optional<Schema.Trimmed>;
|
|
119
|
-
}>, Schema.
|
|
120
|
-
readonly _tag: Schema.Literal<"Url">;
|
|
130
|
+
}>, Schema.TaggedStruct<"Url", {
|
|
121
131
|
readonly url: Schema.Trimmed;
|
|
122
|
-
}>, Schema.
|
|
123
|
-
readonly _tag: Schema.Literal<"Text">;
|
|
132
|
+
}>, Schema.TaggedStruct<"Text", {
|
|
124
133
|
readonly label: Schema.optional<Schema.Trimmed>;
|
|
125
134
|
}>]>;
|
|
126
135
|
readonly status: Schema.Literals<readonly ["processing", "ready", "error"]>;
|
|
@@ -142,5 +151,5 @@ declare const ExtractedKnowledgeDocumentSchema: Schema.Struct<{
|
|
|
142
151
|
type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>;
|
|
143
152
|
declare const defaultKnowledgeChunkMaxTokens = 512;
|
|
144
153
|
//#endregion
|
|
145
|
-
export { CreateKnowledgeDocumentInput, CreateKnowledgeDocumentInputSchema, ExtractedKnowledgeDocument, ExtractedKnowledgeDocumentSchema, IndexedKnowledgeDocument, IndexedKnowledgeDocumentSchema, KnowledgeAvailability, KnowledgeAvailabilitySchema, KnowledgeChunk, KnowledgeChunkSchema, KnowledgeDocument, KnowledgeDocumentSchema, KnowledgeDocumentStatus, KnowledgeDocumentStatusSchema, KnowledgeFile, KnowledgeFileSchema, KnowledgeMetadata, KnowledgeMetadataSchema, KnowledgeScope, KnowledgeScopeSchema, KnowledgeSearchScope, KnowledgeSearchScopeSchema, KnowledgeSource, KnowledgeSourceSchema, NonEmptyTrimmedString, NonNegativeInteger, PositiveInteger, UpdateKnowledgeDocumentInput, UpdateKnowledgeDocumentInputSchema, defaultKnowledgeChunkMaxTokens };
|
|
154
|
+
export { CreateKnowledgeDocumentInput, CreateKnowledgeDocumentInputSchema, ExtractedKnowledgeDocument, ExtractedKnowledgeDocumentSchema, IndexedKnowledgeDocument, IndexedKnowledgeDocumentSchema, KnowledgeAvailability, KnowledgeAvailabilitySchema, KnowledgeChunk, KnowledgeChunkSchema, KnowledgeDocument, KnowledgeDocumentSchema, KnowledgeDocumentStatus, KnowledgeDocumentStatusSchema, KnowledgeFile, KnowledgeFileSchema, KnowledgeFileSource, KnowledgeMetadata, KnowledgeMetadataSchema, KnowledgeScope, KnowledgeScopeSchema, KnowledgeSearchScope, KnowledgeSearchScopeSchema, KnowledgeSearchScopes, KnowledgeSource, KnowledgeSourceSchema, KnowledgeTextSource, KnowledgeUrlSource, NonEmptyTrimmedString, NonNegativeInteger, PositiveInteger, UpdateKnowledgeDocumentInput, UpdateKnowledgeDocumentInputSchema, defaultKnowledgeChunkMaxTokens };
|
|
146
155
|
//# sourceMappingURL=documents.d.mts.map
|
package/dist/documents.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"documents.d.mts","names":[],"sources":["../src/documents.ts"],"mappings":";;;cAEa,qBAAA,EAAqB,MAAA,CAAA,OAAyD;AAAA,
|
|
1
|
+
{"version":3,"file":"documents.d.mts","names":[],"sources":["../src/documents.ts"],"mappings":";;;cAEa,qBAAA,EAAqB,MAAA,CAAA,OAAyD;AAAA,cAE9E,kBAAA,EAAkB,MAAA,CAAA,GAAkE;AAAA,cAEpF,eAAA,EAAe,MAAA,CAAA,GAAyD;AAAA,cAExE,uBAAA,EAAuB,MAAA,CAAA,OAAA,CAAA,MAAA,CAAA,MAAA,EAAA,MAAA,CAAA,OAAA;AAAA,KAExB,iBAAA,GAAoB,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,uBAAA;AAAA,cAE7C,2BAAA,EAA2B,MAAA,CAAA,QAAA;AAAA,KAE5B,qBAAA,GAAwB,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,2BAAA;AAAA,cAEjD,6BAAA,EAA6B,MAAA,CAAA,QAAA;AAAA,KAE9B,uBAAA,GAA0B,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,6BAAA;AAAA,cAEnD,oBAAA,EAAoB,MAAA,CAAA,MAAA;EAAA;;;KAKrB,cAAA,GAAiB,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,oBAAA;AAAA,cAE1C,uBAAA,EAAuB,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;KAiBxB,iBAAA,GAAoB,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,uBAAA;AAAA,cAE7C,kCAAA,EAAkC,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;KAYnC,4BAAA,GAA+B,MAAA,CAAO,MAAA,CAAO,IAAI,QACpD,kCAAA;AAAA,cAGI,kCAAA,EAAkC,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;KAgBnC,4BAAA,GAA+B,MAAA,CAAO,MAAA,CAAO,IAAI,QACpD,kCAAA;AAAA,cAGI,mBAAA,EAAmB,MAAA,CAAA,MAAA;EAAA;;;;;;;;;KAWpB,aAAA,GAAgB,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,mBAAA;AAAA,cAEzC,oBAAA,EAAoB,MAAA,CAAA,MAAA;EAAA;;;;;;;;KAUrB,cAAA,GAAiB,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,oBAAA;AAAA,cAE1C,oBAAA,EAAoB,MAAA,CAAA,YAAA;EAAA,aAE/B,MAAA,CAAA,OAAA;AAAA;AAAA,cAEW,qBAAA,EAAqB,MAAA,CAAA,YAAA;EAAA;;cAIrB,0BAAA,EAA0B,MAAA,CAAA,KAAA,WAAA,MAAA,CAAA,YAAA;EAAA;;;;KAK3B,oBAAA,UAA8B,0BAAA,CAA2B,IAAI;AAAA,cAE5D,mBAAA,EAAmB,MAAA,CAAA,YAAA;EAAA;;;;cAMnB,kBAAA,EAAkB,MAAA,CAAA,YAAA;EAAA,cAE7B,MAAA,CAAA,OAAA;AAAA;AAAA,cAEW,mBAAA,EAAmB,MAAA,CAAA,YAAA;EAAA;;cAInB,qBAAA,EAAqB,MAAA,CAAA,KAAA,WAAA,MAAA,CAAA,YAAA;EAAA;;;;;;;;KAMtB,eAAA,UAAyB,qBAAA,CAAsB,IAAI;AAAA,cAElD,8BAAA,EAA8B,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;KAc/B,wBAAA,GAA2B,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,8BAAA;AAAA,cAEpD,gCAAA,EAAgC,MAAA,CAAA,MAAA;EAAA;;;;;KAOjC,0BAAA,GAA6B,MAAA,CAAO,MAAA,CAAO,IAAI,QAAQ,gCAAA;AAAA,cAEtD,8BAAA"}
|
package/dist/documents.mjs
CHANGED
|
@@ -79,28 +79,20 @@ const KnowledgeChunkSchema = Schema.Struct({
|
|
|
79
79
|
tokenCount: PositiveInteger,
|
|
80
80
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
81
81
|
});
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
82
|
+
const KnowledgeSearchScope = Schema.TaggedStruct("KnowledgeScope", { id: NonEmptyTrimmedString });
|
|
83
|
+
const KnowledgeSearchScopes = Schema.TaggedStruct("KnowledgeScopes", { ids: Schema.NonEmptyArray(NonEmptyTrimmedString) });
|
|
84
|
+
const KnowledgeSearchScopeSchema = Schema.Union([KnowledgeSearchScope, KnowledgeSearchScopes]);
|
|
85
|
+
const KnowledgeFileSource = Schema.TaggedStruct("File", {
|
|
86
|
+
ref: NonEmptyTrimmedString,
|
|
87
|
+
name: Schema.optional(NonEmptyTrimmedString),
|
|
88
|
+
mediaType: Schema.optional(NonEmptyTrimmedString)
|
|
89
|
+
});
|
|
90
|
+
const KnowledgeUrlSource = Schema.TaggedStruct("Url", { url: NonEmptyTrimmedString });
|
|
91
|
+
const KnowledgeTextSource = Schema.TaggedStruct("Text", { label: Schema.optional(NonEmptyTrimmedString) });
|
|
89
92
|
const KnowledgeSourceSchema = Schema.Union([
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
name: Schema.optional(NonEmptyTrimmedString),
|
|
94
|
-
mediaType: Schema.optional(NonEmptyTrimmedString)
|
|
95
|
-
}),
|
|
96
|
-
Schema.Struct({
|
|
97
|
-
_tag: Schema.Literal("Url"),
|
|
98
|
-
url: NonEmptyTrimmedString
|
|
99
|
-
}),
|
|
100
|
-
Schema.Struct({
|
|
101
|
-
_tag: Schema.Literal("Text"),
|
|
102
|
-
label: Schema.optional(NonEmptyTrimmedString)
|
|
103
|
-
})
|
|
93
|
+
KnowledgeFileSource,
|
|
94
|
+
KnowledgeUrlSource,
|
|
95
|
+
KnowledgeTextSource
|
|
104
96
|
]);
|
|
105
97
|
const IndexedKnowledgeDocumentSchema = Schema.Struct({
|
|
106
98
|
id: NonEmptyTrimmedString,
|
|
@@ -123,6 +115,6 @@ const ExtractedKnowledgeDocumentSchema = Schema.Struct({
|
|
|
123
115
|
});
|
|
124
116
|
const defaultKnowledgeChunkMaxTokens = 512;
|
|
125
117
|
//#endregion
|
|
126
|
-
export { CreateKnowledgeDocumentInputSchema, ExtractedKnowledgeDocumentSchema, IndexedKnowledgeDocumentSchema, KnowledgeAvailabilitySchema, KnowledgeChunkSchema, KnowledgeDocumentSchema, KnowledgeDocumentStatusSchema, KnowledgeFileSchema, KnowledgeMetadataSchema, KnowledgeScopeSchema, KnowledgeSearchScopeSchema, KnowledgeSourceSchema, NonEmptyTrimmedString, NonNegativeInteger, PositiveInteger, UpdateKnowledgeDocumentInputSchema, defaultKnowledgeChunkMaxTokens };
|
|
118
|
+
export { CreateKnowledgeDocumentInputSchema, ExtractedKnowledgeDocumentSchema, IndexedKnowledgeDocumentSchema, KnowledgeAvailabilitySchema, KnowledgeChunkSchema, KnowledgeDocumentSchema, KnowledgeDocumentStatusSchema, KnowledgeFileSchema, KnowledgeFileSource, KnowledgeMetadataSchema, KnowledgeScopeSchema, KnowledgeSearchScope, KnowledgeSearchScopeSchema, KnowledgeSearchScopes, KnowledgeSourceSchema, KnowledgeTextSource, KnowledgeUrlSource, NonEmptyTrimmedString, NonNegativeInteger, PositiveInteger, UpdateKnowledgeDocumentInputSchema, defaultKnowledgeChunkMaxTokens };
|
|
127
119
|
|
|
128
120
|
//# sourceMappingURL=documents.mjs.map
|
package/dist/documents.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"documents.mjs","names":[],"sources":["../src/documents.ts"],"sourcesContent":["import * as Schema from 'effect/Schema'\n\nexport const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))\nexport const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))\nexport const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))\n\nexport const KnowledgeMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)\nexport type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>\n\nexport const KnowledgeAvailabilitySchema = Schema.Literals(['pinned', 'searchable', 'archived'])\nexport type KnowledgeAvailability = Schema.Schema.Type<typeof KnowledgeAvailabilitySchema>\n\nexport const KnowledgeDocumentStatusSchema = Schema.Literals(['processing', 'ready', 'error'])\nexport type KnowledgeDocumentStatus = Schema.Schema.Type<typeof KnowledgeDocumentStatusSchema>\n\nexport const KnowledgeScopeSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n kind: Schema.optional(NonEmptyTrimmedString)\n})\nexport type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>\n\nexport const KnowledgeDocumentSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n slug: NonEmptyTrimmedString,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n status: KnowledgeDocumentStatusSchema,\n availability: KnowledgeAvailabilitySchema,\n summary: Schema.optional(Schema.String),\n errorMessage: Schema.optional(Schema.String),\n reviewedAt: Schema.optional(Schema.DateTimeUtc),\n metadata: Schema.optional(KnowledgeMetadataSchema),\n createdAt: Schema.DateTimeUtc,\n updatedAt: Schema.DateTimeUtc\n})\nexport type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>\n\nexport const CreateKnowledgeDocumentInputSchema = Schema.Struct({\n scope: KnowledgeScopeSchema,\n slug: NonEmptyTrimmedString,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema,\n summary: Schema.optional(Schema.String),\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\nexport type CreateKnowledgeDocumentInput = Schema.Schema.Type<\n typeof CreateKnowledgeDocumentInputSchema\n>\n\nexport const UpdateKnowledgeDocumentInputSchema = Schema.Struct({\n scope: KnowledgeScopeSchema,\n id: NonEmptyTrimmedString,\n slug: Schema.optional(NonEmptyTrimmedString),\n title: Schema.optional(NonEmptyTrimmedString),\n purpose: Schema.optional(NonEmptyTrimmedString),\n origin: Schema.optional(NonEmptyTrimmedString),\n content: Schema.optional(NonEmptyTrimmedString),\n status: Schema.optional(KnowledgeDocumentStatusSchema),\n availability: Schema.optional(KnowledgeAvailabilitySchema),\n summary: Schema.optional(Schema.String),\n errorMessage: Schema.optional(Schema.String),\n reviewedAt: Schema.optional(Schema.DateTimeUtc),\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\nexport type UpdateKnowledgeDocumentInput = Schema.Schema.Type<\n typeof UpdateKnowledgeDocumentInputSchema\n>\n\nexport const KnowledgeFileSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n documentId: NonEmptyTrimmedString,\n storageKey: NonEmptyTrimmedString,\n mediaType: Schema.optional(NonEmptyTrimmedString),\n byteSize: Schema.optional(NonNegativeInteger),\n checksum: Schema.optional(NonEmptyTrimmedString),\n metadata: Schema.optional(KnowledgeMetadataSchema),\n createdAt: Schema.DateTimeUtc\n})\nexport type KnowledgeFile = Schema.Schema.Type<typeof KnowledgeFileSchema>\n\nexport const KnowledgeChunkSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n scopeId: NonEmptyTrimmedString,\n documentId: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n position: NonNegativeInteger,\n tokenCount: PositiveInteger,\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\nexport type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>\n\nexport const
|
|
1
|
+
{"version":3,"file":"documents.mjs","names":[],"sources":["../src/documents.ts"],"sourcesContent":["import * as Schema from 'effect/Schema'\n\nexport const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))\n\nexport const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))\n\nexport const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))\n\nexport const KnowledgeMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)\n\nexport type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>\n\nexport const KnowledgeAvailabilitySchema = Schema.Literals(['pinned', 'searchable', 'archived'])\n\nexport type KnowledgeAvailability = Schema.Schema.Type<typeof KnowledgeAvailabilitySchema>\n\nexport const KnowledgeDocumentStatusSchema = Schema.Literals(['processing', 'ready', 'error'])\n\nexport type KnowledgeDocumentStatus = Schema.Schema.Type<typeof KnowledgeDocumentStatusSchema>\n\nexport const KnowledgeScopeSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n kind: Schema.optional(NonEmptyTrimmedString)\n})\n\nexport type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>\n\nexport const KnowledgeDocumentSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n slug: NonEmptyTrimmedString,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n status: KnowledgeDocumentStatusSchema,\n availability: KnowledgeAvailabilitySchema,\n summary: Schema.optional(Schema.String),\n errorMessage: Schema.optional(Schema.String),\n reviewedAt: Schema.optional(Schema.DateTimeUtc),\n metadata: Schema.optional(KnowledgeMetadataSchema),\n createdAt: Schema.DateTimeUtc,\n updatedAt: Schema.DateTimeUtc\n})\n\nexport type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>\n\nexport const CreateKnowledgeDocumentInputSchema = Schema.Struct({\n scope: KnowledgeScopeSchema,\n slug: NonEmptyTrimmedString,\n title: NonEmptyTrimmedString,\n purpose: NonEmptyTrimmedString,\n origin: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n availability: KnowledgeAvailabilitySchema,\n summary: Schema.optional(Schema.String),\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\n\nexport type CreateKnowledgeDocumentInput = Schema.Schema.Type<\n typeof CreateKnowledgeDocumentInputSchema\n>\n\nexport const UpdateKnowledgeDocumentInputSchema = Schema.Struct({\n scope: KnowledgeScopeSchema,\n id: NonEmptyTrimmedString,\n slug: Schema.optional(NonEmptyTrimmedString),\n title: Schema.optional(NonEmptyTrimmedString),\n purpose: Schema.optional(NonEmptyTrimmedString),\n origin: Schema.optional(NonEmptyTrimmedString),\n content: Schema.optional(NonEmptyTrimmedString),\n status: Schema.optional(KnowledgeDocumentStatusSchema),\n availability: Schema.optional(KnowledgeAvailabilitySchema),\n summary: Schema.optional(Schema.String),\n errorMessage: Schema.optional(Schema.String),\n reviewedAt: Schema.optional(Schema.DateTimeUtc),\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\n\nexport type UpdateKnowledgeDocumentInput = Schema.Schema.Type<\n typeof UpdateKnowledgeDocumentInputSchema\n>\n\nexport const KnowledgeFileSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n documentId: NonEmptyTrimmedString,\n storageKey: NonEmptyTrimmedString,\n mediaType: Schema.optional(NonEmptyTrimmedString),\n byteSize: Schema.optional(NonNegativeInteger),\n checksum: Schema.optional(NonEmptyTrimmedString),\n metadata: Schema.optional(KnowledgeMetadataSchema),\n createdAt: Schema.DateTimeUtc\n})\n\nexport type KnowledgeFile = Schema.Schema.Type<typeof KnowledgeFileSchema>\n\nexport const KnowledgeChunkSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n scopeId: NonEmptyTrimmedString,\n documentId: NonEmptyTrimmedString,\n content: NonEmptyTrimmedString,\n position: NonNegativeInteger,\n tokenCount: PositiveInteger,\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\n\nexport type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>\n\nexport const KnowledgeSearchScope = Schema.TaggedStruct('KnowledgeScope', {\n id: NonEmptyTrimmedString\n})\n\nexport const KnowledgeSearchScopes = Schema.TaggedStruct('KnowledgeScopes', {\n ids: Schema.NonEmptyArray(NonEmptyTrimmedString)\n})\n\nexport const KnowledgeSearchScopeSchema = Schema.Union([\n KnowledgeSearchScope,\n KnowledgeSearchScopes\n])\n\nexport type KnowledgeSearchScope = typeof KnowledgeSearchScopeSchema.Type\n\nexport const KnowledgeFileSource = Schema.TaggedStruct('File', {\n ref: NonEmptyTrimmedString,\n name: Schema.optional(NonEmptyTrimmedString),\n mediaType: Schema.optional(NonEmptyTrimmedString)\n})\n\nexport const KnowledgeUrlSource = Schema.TaggedStruct('Url', {\n url: NonEmptyTrimmedString\n})\n\nexport const KnowledgeTextSource = Schema.TaggedStruct('Text', {\n label: Schema.optional(NonEmptyTrimmedString)\n})\n\nexport const KnowledgeSourceSchema = Schema.Union([\n KnowledgeFileSource,\n KnowledgeUrlSource,\n KnowledgeTextSource\n])\n\nexport type KnowledgeSource = typeof KnowledgeSourceSchema.Type\n\nexport const IndexedKnowledgeDocumentSchema = Schema.Struct({\n id: NonEmptyTrimmedString,\n scopeId: NonEmptyTrimmedString,\n source: KnowledgeSourceSchema,\n status: KnowledgeDocumentStatusSchema,\n title: Schema.optional(NonEmptyTrimmedString),\n summary: Schema.optional(Schema.String),\n errorMessage: Schema.optional(Schema.String),\n contentHash: Schema.optional(NonEmptyTrimmedString),\n tokenCount: Schema.optional(NonNegativeInteger),\n chunkCount: Schema.optional(NonNegativeInteger),\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\n\nexport type IndexedKnowledgeDocument = Schema.Schema.Type<typeof IndexedKnowledgeDocumentSchema>\n\nexport const ExtractedKnowledgeDocumentSchema = Schema.Struct({\n content: NonEmptyTrimmedString,\n title: Schema.optional(NonEmptyTrimmedString),\n summary: Schema.optional(Schema.String),\n metadata: Schema.optional(KnowledgeMetadataSchema)\n})\n\nexport type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>\n\nexport const defaultKnowledgeChunkMaxTokens = 512\n"],"mappings":";;AAEA,MAAa,wBAAwB,OAAO,QAAQ,KAAK,OAAO,MAAM,OAAO,WAAW,CAAC,CAAC;AAE1F,MAAa,qBAAqB,OAAO,IAAI,KAAK,OAAO,MAAM,OAAO,uBAAuB,CAAC,CAAC,CAAC;AAEhG,MAAa,kBAAkB,OAAO,IAAI,KAAK,OAAO,MAAM,OAAO,cAAc,CAAC,CAAC,CAAC;AAEpF,MAAa,0BAA0B,OAAO,OAAO,OAAO,QAAQ,OAAO,OAAO;AAIlF,MAAa,8BAA8B,OAAO,SAAS;CAAC;CAAU;CAAc;AAAU,CAAC;AAI/F,MAAa,gCAAgC,OAAO,SAAS;CAAC;CAAc;CAAS;AAAO,CAAC;AAI7F,MAAa,uBAAuB,OAAO,OAAO;CAChD,IAAI;CACJ,MAAM,OAAO,SAAS,qBAAqB;AAC7C,CAAC;AAID,MAAa,0BAA0B,OAAO,OAAO;CACnD,IAAI;CACJ,MAAM;CACN,OAAO;CACP,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,cAAc;CACd,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,cAAc,OAAO,SAAS,OAAO,MAAM;CAC3C,YAAY,OAAO,SAAS,OAAO,WAAW;CAC9C,UAAU,OAAO,SAAS,uBAAuB;CACjD,WAAW,OAAO;CAClB,WAAW,OAAO;AACpB,CAAC;AAID,MAAa,qCAAqC,OAAO,OAAO;CAC9D,OAAO;CACP,MAAM;CACN,OAAO;CACP,SAAS;CACT,QAAQ;CACR,SAAS;CACT,cAAc;CACd,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,UAAU,OAAO,SAAS,uBAAuB;AACnD,CAAC;AAMD,MAAa,qCAAqC,OAAO,OAAO;CAC9D,OAAO;CACP,IAAI;CACJ,MAAM,OAAO,SAAS,qBAAqB;CAC3C,OAAO,OAAO,SAAS,qBAAqB;CAC5C,SAAS,OAAO,SAAS,qBAAqB;CAC9C,QAAQ,OAAO,SAAS,qBAAqB;CAC7C,SAAS,OAAO,SAAS,qBAAqB;CAC9C,QAAQ,OAAO,SAAS,6BAA6B;CACrD,cAAc,OAAO,SAAS,2BAA2B;CACzD,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,cAAc,OAAO,SAAS,OAAO,MAAM;CAC3C,YAAY,OAAO,SAAS,OAAO,WAAW;CAC9C,UAAU,OAAO,SAAS,uBAAuB;AACnD,CAAC;AAMD,MAAa,sBAAsB,OAAO,OAAO;CAC/C,IAAI;CACJ,YAAY;CACZ,YAAY;CACZ,WAAW,OAAO,SAAS,qBAAqB;CAChD,UAAU,OAAO,SAAS,kBAAkB;CAC5C,UAAU,OAAO,SAAS,qBAAqB;CAC/C,UAAU,OAAO,SAAS,uBAAuB;CACjD,WAAW,OAAO;AACpB,CAAC;AAID,MAAa,uBAAuB,OAAO,OAAO;CAChD,IAAI;CACJ,SAAS;CACT,YAAY;CACZ,SAAS;CACT,UAAU;CACV,YAAY;CACZ,UAAU,OAAO,SAAS,uBAAuB;AACnD,CAAC;AAID,MAAa,uBAAuB,OAAO,aAAa,kBAAkB,EACxE,IAAI,sBACN,CAAC;AAED,MAAa,wBAAwB,OAAO,aAAa,mBAAmB,EAC1E,KAAK,OAAO,cAAc,qBAAqB,EACjD,CAAC;AAED,MAAa,6BAA6B,OAAO,MAAM,CACrD,sBACA,qBACF,CAAC;AAID,MAAa,sBAAsB,OAAO,aAAa,QAAQ;CAC7D,KAAK;CACL,MAAM,OAAO,SAAS,qBAAqB;CAC3C,WAAW,OAAO,SAAS,qBAAqB;AAClD,CAAC;AAED,MAAa,qBAAqB,OAAO,aAAa,OAAO,EAC3D,KAAK,sBACP,CAAC;AAED,MAAa,sBAAsB,OAAO,aAAa,QAAQ,EAC7D,OAAO,OAAO,SAAS,qBAAqB,EAC9C,CAAC;AAED,MAAa,wBAAwB,OAAO,MAAM;CAChD;CACA;CACA;AACF,CAAC;AAID,MAAa,iCAAiC,OAAO,OAAO;CAC1D,IAAI;CACJ,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,OAAO,OAAO,SAAS,qBAAqB;CAC5C,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,cAAc,OAAO,SAAS,OAAO,MAAM;CAC3C,aAAa,OAAO,SAAS,qBAAqB;CAClD,YAAY,OAAO,SAAS,kBAAkB;CAC9C,YAAY,OAAO,SAAS,kBAAkB;CAC9C,UAAU,OAAO,SAAS,uBAAuB;AACnD,CAAC;AAID,MAAa,mCAAmC,OAAO,OAAO;CAC5D,SAAS;CACT,OAAO,OAAO,SAAS,qBAAqB;CAC5C,SAAS,OAAO,SAAS,OAAO,MAAM;CACtC,UAAU,OAAO,SAAS,uBAAuB;AACnD,CAAC;AAID,MAAa,iCAAiC"}
|
package/dist/errors.mjs
CHANGED
|
@@ -8,15 +8,15 @@ var KnowledgeEmbeddingError = class extends Data.TaggedError("KnowledgeEmbedding
|
|
|
8
8
|
var KnowledgeSummarizationError = class extends Data.TaggedError("KnowledgeSummarizationError") {};
|
|
9
9
|
var KnowledgeIngestionError = class extends Data.TaggedError("KnowledgeIngestionError") {};
|
|
10
10
|
var KnowledgeSearchError = class extends Data.TaggedError("KnowledgeSearchError") {};
|
|
11
|
-
var KnowledgeStoreError = class extends Schema.
|
|
11
|
+
var KnowledgeStoreError = class extends Schema.TaggedError()("KnowledgeStoreError", {
|
|
12
12
|
message: Schema.String,
|
|
13
13
|
cause: Schema.optional(Schema.Unknown)
|
|
14
14
|
}) {};
|
|
15
|
-
var KnowledgeFileError = class extends Schema.
|
|
15
|
+
var KnowledgeFileError = class extends Schema.TaggedError()("KnowledgeFileError", {
|
|
16
16
|
message: Schema.String,
|
|
17
17
|
cause: Schema.optional(Schema.Unknown)
|
|
18
18
|
}) {};
|
|
19
|
-
var KnowledgeContextError = class extends Schema.
|
|
19
|
+
var KnowledgeContextError = class extends Schema.TaggedError()("KnowledgeContextError", {
|
|
20
20
|
message: Schema.String,
|
|
21
21
|
cause: Schema.optional(Schema.Unknown)
|
|
22
22
|
}) {};
|
package/dist/errors.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.mjs","names":[],"sources":["../src/errors.ts"],"sourcesContent":["import { Data } from 'effect'\nimport * as Schema from 'effect/Schema'\n\nexport class SearchIndexStoreError extends Data.TaggedError('SearchIndexStoreError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeExtractionError extends Data.TaggedError('KnowledgeExtractionError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeChunkingError extends Data.TaggedError('KnowledgeChunkingError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeEmbeddingError extends Data.TaggedError('KnowledgeEmbeddingError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeSummarizationError extends Data.TaggedError('KnowledgeSummarizationError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeIngestionError extends Data.TaggedError('KnowledgeIngestionError')<{\n readonly message: string\n readonly stage: 'store' | 'extract' | 'chunk' | 'embed' | 'summarize'\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeSearchError extends Data.TaggedError('KnowledgeSearchError')<{\n readonly message: string\n readonly stage: 'store' | 'embed'\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeStoreError extends Schema.
|
|
1
|
+
{"version":3,"file":"errors.mjs","names":[],"sources":["../src/errors.ts"],"sourcesContent":["import { Data } from 'effect'\nimport * as Schema from 'effect/Schema'\n\nexport class SearchIndexStoreError extends Data.TaggedError('SearchIndexStoreError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeExtractionError extends Data.TaggedError('KnowledgeExtractionError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeChunkingError extends Data.TaggedError('KnowledgeChunkingError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeEmbeddingError extends Data.TaggedError('KnowledgeEmbeddingError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeSummarizationError extends Data.TaggedError('KnowledgeSummarizationError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeIngestionError extends Data.TaggedError('KnowledgeIngestionError')<{\n readonly message: string\n readonly stage: 'store' | 'extract' | 'chunk' | 'embed' | 'summarize'\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeSearchError extends Data.TaggedError('KnowledgeSearchError')<{\n readonly message: string\n readonly stage: 'store' | 'embed'\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeStoreError extends Schema.TaggedError<KnowledgeStoreError>()(\n 'KnowledgeStoreError',\n {\n message: Schema.String,\n cause: Schema.optional(Schema.Unknown)\n }\n) {}\n\nexport class KnowledgeFileError extends Schema.TaggedError<KnowledgeFileError>()(\n 'KnowledgeFileError',\n {\n message: Schema.String,\n cause: Schema.optional(Schema.Unknown)\n }\n) {}\n\nexport class KnowledgeContextError extends Schema.TaggedError<KnowledgeContextError>()(\n 'KnowledgeContextError',\n {\n message: Schema.String,\n cause: Schema.optional(Schema.Unknown)\n }\n) {}\n"],"mappings":";;;AAGA,IAAa,wBAAb,cAA2C,KAAK,YAAY,uBAAuB,EAGhF,CAAC;AAEJ,IAAa,2BAAb,cAA8C,KAAK,YAAY,0BAA0B,EAGtF,CAAC;AAEJ,IAAa,yBAAb,cAA4C,KAAK,YAAY,wBAAwB,EAGlF,CAAC;AAEJ,IAAa,0BAAb,cAA6C,KAAK,YAAY,yBAAyB,EAGpF,CAAC;AAEJ,IAAa,8BAAb,cAAiD,KAAK,YAAY,6BAA6B,EAG5F,CAAC;AAEJ,IAAa,0BAAb,cAA6C,KAAK,YAAY,yBAAyB,EAIpF,CAAC;AAEJ,IAAa,uBAAb,cAA0C,KAAK,YAAY,sBAAsB,EAI9E,CAAC;AAEJ,IAAa,sBAAb,cAAyC,OAAO,YAAiC,EAC/E,uBACA;CACE,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CACF,EAAE,CAAC;AAEH,IAAa,qBAAb,cAAwC,OAAO,YAAgC,EAC7E,sBACA;CACE,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CACF,EAAE,CAAC;AAEH,IAAa,wBAAb,cAA2C,OAAO,YAAmC,EACnF,yBACA;CACE,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CACF,EAAE,CAAC"}
|
package/dist/ingestion.d.mts
CHANGED
|
@@ -17,11 +17,12 @@ type IngestKnowledgeDocumentInput = {
|
|
|
17
17
|
declare const ingestKnowledgeDocument: (input: IngestKnowledgeDocumentInput) => Effect.Effect<{
|
|
18
18
|
readonly id: string;
|
|
19
19
|
readonly scopeId: string;
|
|
20
|
+
readonly status: "processing" | "ready" | "error";
|
|
20
21
|
readonly source: {
|
|
21
22
|
readonly _tag: "File";
|
|
22
23
|
readonly ref: string;
|
|
23
|
-
readonly name?: string | undefined;
|
|
24
24
|
readonly mediaType?: string | undefined;
|
|
25
|
+
readonly name?: string | undefined;
|
|
25
26
|
} | {
|
|
26
27
|
readonly _tag: "Url";
|
|
27
28
|
readonly url: string;
|
|
@@ -29,17 +30,16 @@ declare const ingestKnowledgeDocument: (input: IngestKnowledgeDocumentInput) =>
|
|
|
29
30
|
readonly _tag: "Text";
|
|
30
31
|
readonly label?: string | undefined;
|
|
31
32
|
};
|
|
32
|
-
readonly
|
|
33
|
-
|
|
33
|
+
readonly metadata?: {
|
|
34
|
+
readonly [x: string]: unknown;
|
|
35
|
+
} | undefined;
|
|
36
|
+
readonly tokenCount?: number | undefined;
|
|
34
37
|
readonly summary?: string | undefined;
|
|
35
38
|
readonly errorMessage?: string | undefined;
|
|
39
|
+
readonly title?: string | undefined;
|
|
36
40
|
readonly contentHash?: string | undefined;
|
|
37
|
-
readonly tokenCount?: number | undefined;
|
|
38
41
|
readonly chunkCount?: number | undefined;
|
|
39
|
-
|
|
40
|
-
readonly [x: string]: unknown;
|
|
41
|
-
} | undefined;
|
|
42
|
-
}, KnowledgeIngestionError, KnowledgeSummarizer | SearchIndexStore | KnowledgeEmbedder | KnowledgeChunker | KnowledgeExtractor>;
|
|
42
|
+
}, KnowledgeIngestionError, KnowledgeSummarizer | KnowledgeChunker | KnowledgeEmbedder | KnowledgeExtractor | SearchIndexStore>;
|
|
43
43
|
type KnowledgeIngestionPipeline = {
|
|
44
44
|
readonly ingest: (input: IngestKnowledgeDocumentInput) => ReturnType<typeof ingestKnowledgeDocument>;
|
|
45
45
|
};
|
package/dist/ingestion.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion.d.mts","names":[],"sources":["../src/ingestion.ts"],"mappings":";;;;;;;;;KAUY,4BAAA;EAAA,SACD,OAAA;EAAA,SACA,UAAA;EAAA,SACA,MAAA,EAAQ,qBAAqB;EAAA,SAC7B,WAAA;EAAA,SACA,SAAA;AAAA;AAAA,cAaE,uBAAA,GAA2B,KAAA,EAAO,4BAAA,KAA4B,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"ingestion.d.mts","names":[],"sources":["../src/ingestion.ts"],"mappings":";;;;;;;;;KAUY,4BAAA;EAAA,SACD,OAAA;EAAA,SACA,UAAA;EAAA,SACA,MAAA,EAAQ,qBAAqB;EAAA,SAC7B,WAAA;EAAA,SACA,SAAA;AAAA;AAAA,cAaE,uBAAA,GAA2B,KAAA,EAAO,4BAAA,KAA4B,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;KA0I/D,0BAAA;EAAA,SACD,MAAA,GACP,KAAA,EAAO,4BAAA,KACJ,UAAA,QAAkB,uBAAA;AAAA;AAAA,cAGZ,qBAAA,QAA4B,0BAEvC"}
|
package/dist/ingestion.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { defaultKnowledgeChunkMaxTokens } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type IngestKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly source: LoadedKnowledgeSource\n readonly contentHash?: string\n readonly maxTokens?: number\n}\n\nconst markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n yield* store.markDocumentError({\n scopeId: input.scopeId,\n documentId: input.documentId,\n message: error.message\n })\n }).pipe(Effect.catch(() => Effect.void))\n\nexport const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>\n Effect.gen(function* () {\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.scope_id': input.scopeId,\n 'knowledge_search.document_id': input.documentId,\n 'knowledge_search.source_type': input.source.source._tag\n })\n const store = yield* SearchIndexStore\n const extractor = yield* KnowledgeExtractor\n const chunker = yield* KnowledgeChunker\n const embedder = yield* KnowledgeEmbedder\n const summarizer = yield* KnowledgeSummarizer\n\n yield* store\n .upsertDocument({\n document: {\n id: input.documentId,\n scopeId: input.scopeId,\n source: input.source.source,\n status: 'processing',\n metadata: input.source.metadata\n }\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const extracted = yield* extractor\n .extract(input.source)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })\n )\n )\n\n const chunks = yield* chunker\n .chunk({\n scopeId: input.scopeId,\n documentId: input.documentId,\n content: extracted.content,\n maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })\n )\n )\n\n const indexed = yield* Effect.all(\n {\n embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'embed',\n cause: error\n })\n )\n ),\n summary: summarizer\n .summarize({\n content: extracted.content,\n sourceTitle: extracted.title,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'summarize',\n cause: error\n })\n )\n )\n },\n { concurrency: 'unbounded' }\n )\n\n if (indexed.embeddings.length !== chunks.length) {\n return yield* Effect.fail(\n new KnowledgeIngestionError({\n message: 'Embedding count did not match chunk count',\n stage: 'embed'\n })\n )\n }\n\n const indexedChunks = Arr.zip(chunks, indexed.embeddings).map(([chunk, embedding]) => ({\n chunk,\n embedding\n }))\n yield* store\n .replaceDocumentChunks({\n scopeId: input.scopeId,\n documentId: input.documentId,\n chunks: indexedChunks\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)\n return yield* store\n .markDocumentReady({\n scopeId: input.scopeId,\n documentId: input.documentId,\n title: indexed.summary.title ?? extracted.title,\n summary: indexed.summary.summary ?? extracted.summary,\n contentHash: input.contentHash,\n tokenCount,\n chunkCount: chunks.length\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n }).pipe(\n Effect.withSpan('knowledge_search.ingestDocument'),\n Effect.catch(error =>\n markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))\n )\n )\n\nexport type KnowledgeIngestionPipeline = {\n readonly ingest: (\n input: IngestKnowledgeDocumentInput\n ) => ReturnType<typeof ingestKnowledgeDocument>\n}\n\nexport const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({\n ingest: ingestKnowledgeDocument\n})\n"],"mappings":";;;;;;;;;AAkBA,MAAM,uBAAuB,OAAqC,UAChE,OAAO,IAAI,aAAa;CAEtB,QAAO,OADc,kBACR,kBAAkB;EAC7B,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,MAAM;CACjB,CAAC;AACH,CAAC,EAAE,KAAK,OAAO,YAAY,OAAO,IAAI,CAAC;AAEzC,MAAa,2BAA2B,UACtC,OAAO,IAAI,aAAa;CACtB,OAAO,OAAO,oBAAoB;EAChC,6BAA6B,MAAM;EACnC,gCAAgC,MAAM;EACtC,gCAAgC,MAAM,OAAO,OAAO;CACtD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,YAAY,OAAO;CACzB,MAAM,UAAU,OAAO;CACvB,MAAM,WAAW,OAAO;CACxB,MAAM,aAAa,OAAO;CAE1B,OAAO,MACJ,eAAe,EACd,UAAU;EACR,IAAI,MAAM;EACV,SAAS,MAAM;EACf,QAAQ,MAAM,OAAO;EACrB,QAAQ;EACR,UAAU,MAAM,OAAO;CACzB,EACF,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,YAAY,OAAO,UACtB,QAAQ,MAAM,MAAM,EACpB,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAW,OAAO;CAAM,CAAC,CAC1F,CACF;CAEF,MAAM,SAAS,OAAO,QACnB,MAAM;EACL,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,UAAU;EACnB,WAAW,MAAM,aAAA;EACjB,UAAU,UAAU;CACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,UAAU,OAAO,OAAO,IAC5B;EACE,YAAY,SAAS,WAAW,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,EAAE,KAClE,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;EACA,SAAS,WACN,UAAU;GACT,SAAS,UAAU;GACnB,aAAa,UAAU;GACvB,UAAU,UAAU;EACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;CACJ,GACA,EAAE,aAAa,YAAY,CAC7B;CAEA,IAAI,QAAQ,WAAW,WAAW,OAAO,QACvC,OAAO,OAAO,OAAO,KACnB,IAAI,wBAAwB;EAC1B,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,MAAM,gBAAgBA,MAAI,IAAI,QAAQ,QAAQ,UAAU,EAAE,KAAK,CAAC,OAAO,gBAAgB;EACrF;EACA;CACF,EAAE;
|
|
1
|
+
{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { defaultKnowledgeChunkMaxTokens } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type IngestKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly source: LoadedKnowledgeSource\n readonly contentHash?: string\n readonly maxTokens?: number\n}\n\nconst markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n yield* store.markDocumentError({\n scopeId: input.scopeId,\n documentId: input.documentId,\n message: error.message\n })\n }).pipe(Effect.catch(() => Effect.void))\n\nexport const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>\n Effect.gen(function* () {\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.scope_id': input.scopeId,\n 'knowledge_search.document_id': input.documentId,\n 'knowledge_search.source_type': input.source.source._tag\n })\n const store = yield* SearchIndexStore\n const extractor = yield* KnowledgeExtractor\n const chunker = yield* KnowledgeChunker\n const embedder = yield* KnowledgeEmbedder\n const summarizer = yield* KnowledgeSummarizer\n\n yield* store\n .upsertDocument({\n document: {\n id: input.documentId,\n scopeId: input.scopeId,\n source: input.source.source,\n status: 'processing',\n metadata: input.source.metadata\n }\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const extracted = yield* extractor\n .extract(input.source)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })\n )\n )\n\n const chunks = yield* chunker\n .chunk({\n scopeId: input.scopeId,\n documentId: input.documentId,\n content: extracted.content,\n maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })\n )\n )\n\n const indexed = yield* Effect.all(\n {\n embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'embed',\n cause: error\n })\n )\n ),\n summary: summarizer\n .summarize({\n content: extracted.content,\n sourceTitle: extracted.title,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'summarize',\n cause: error\n })\n )\n )\n },\n { concurrency: 'unbounded' }\n )\n\n if (indexed.embeddings.length !== chunks.length) {\n return yield* Effect.fail(\n new KnowledgeIngestionError({\n message: 'Embedding count did not match chunk count',\n stage: 'embed'\n })\n )\n }\n\n const indexedChunks = Arr.zip(chunks, indexed.embeddings).map(([chunk, embedding]) => ({\n chunk,\n embedding\n }))\n\n yield* store\n .replaceDocumentChunks({\n scopeId: input.scopeId,\n documentId: input.documentId,\n chunks: indexedChunks\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)\n\n return yield* store\n .markDocumentReady({\n scopeId: input.scopeId,\n documentId: input.documentId,\n title: indexed.summary.title ?? extracted.title,\n summary: indexed.summary.summary ?? extracted.summary,\n contentHash: input.contentHash,\n tokenCount,\n chunkCount: chunks.length\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n }).pipe(\n Effect.withSpan('knowledge_search.ingestDocument'),\n Effect.catch(error =>\n markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))\n )\n )\n\nexport type KnowledgeIngestionPipeline = {\n readonly ingest: (\n input: IngestKnowledgeDocumentInput\n ) => ReturnType<typeof ingestKnowledgeDocument>\n}\n\nexport const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({\n ingest: ingestKnowledgeDocument\n})\n"],"mappings":";;;;;;;;;AAkBA,MAAM,uBAAuB,OAAqC,UAChE,OAAO,IAAI,aAAa;CAEtB,QAAO,OADc,kBACR,kBAAkB;EAC7B,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,MAAM;CACjB,CAAC;AACH,CAAC,EAAE,KAAK,OAAO,YAAY,OAAO,IAAI,CAAC;AAEzC,MAAa,2BAA2B,UACtC,OAAO,IAAI,aAAa;CACtB,OAAO,OAAO,oBAAoB;EAChC,6BAA6B,MAAM;EACnC,gCAAgC,MAAM;EACtC,gCAAgC,MAAM,OAAO,OAAO;CACtD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,YAAY,OAAO;CACzB,MAAM,UAAU,OAAO;CACvB,MAAM,WAAW,OAAO;CACxB,MAAM,aAAa,OAAO;CAE1B,OAAO,MACJ,eAAe,EACd,UAAU;EACR,IAAI,MAAM;EACV,SAAS,MAAM;EACf,QAAQ,MAAM,OAAO;EACrB,QAAQ;EACR,UAAU,MAAM,OAAO;CACzB,EACF,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,YAAY,OAAO,UACtB,QAAQ,MAAM,MAAM,EACpB,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAW,OAAO;CAAM,CAAC,CAC1F,CACF;CAEF,MAAM,SAAS,OAAO,QACnB,MAAM;EACL,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,UAAU;EACnB,WAAW,MAAM,aAAA;EACjB,UAAU,UAAU;CACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,UAAU,OAAO,OAAO,IAC5B;EACE,YAAY,SAAS,WAAW,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,EAAE,KAClE,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;EACA,SAAS,WACN,UAAU;GACT,SAAS,UAAU;GACnB,aAAa,UAAU;GACvB,UAAU,UAAU;EACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;CACJ,GACA,EAAE,aAAa,YAAY,CAC7B;CAEA,IAAI,QAAQ,WAAW,WAAW,OAAO,QACvC,OAAO,OAAO,OAAO,KACnB,IAAI,wBAAwB;EAC1B,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,MAAM,gBAAgBA,MAAI,IAAI,QAAQ,QAAQ,UAAU,EAAE,KAAK,CAAC,OAAO,gBAAgB;EACrF;EACA;CACF,EAAE;CAEF,OAAO,MACJ,sBAAsB;EACrB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,QAAQ;CACV,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,aAAa,OAAO,QAAQ,OAAO,UAAU,QAAQ,MAAM,YAAY,CAAC;CAE9E,OAAO,OAAO,MACX,kBAAkB;EACjB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,OAAO,QAAQ,QAAQ,SAAS,UAAU;EAC1C,SAAS,QAAQ,QAAQ,WAAW,UAAU;EAC9C,aAAa,MAAM;EACnB;EACA,YAAY,OAAO;CACrB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;AACJ,CAAC,EAAE,KACD,OAAO,SAAS,iCAAiC,GACjD,OAAO,OAAM,UACX,oBAAoB,OAAO,KAAK,EAAE,KAAK,OAAO,cAAc,OAAO,KAAK,KAAK,CAAC,CAAC,CACjF,CACF;AAQF,MAAa,+BAA2D,EACtE,QAAQ,wBACV"}
|
package/dist/search.d.mts
CHANGED
|
@@ -43,7 +43,7 @@ declare const fuseKnowledgeSearchResults: (input: {
|
|
|
43
43
|
readonly limit: number;
|
|
44
44
|
readonly rankConstant?: number;
|
|
45
45
|
}) => ReadonlyArray<KnowledgeSearchResult>;
|
|
46
|
-
declare const searchKnowledge: (input: KnowledgeSearchInput) => Effect.Effect<readonly KnowledgeSearchResult[], KnowledgeSearchError,
|
|
46
|
+
declare const searchKnowledge: (input: KnowledgeSearchInput) => Effect.Effect<readonly KnowledgeSearchResult[], KnowledgeSearchError, KnowledgeEmbedder | SearchIndexStore>;
|
|
47
47
|
//#endregion
|
|
48
48
|
export { KnowledgeSearchContext, KnowledgeSearchInput, KnowledgeSearchMode, KnowledgeSearchResult, KnowledgeSearchScores, KnowledgeSearcher, fuseKnowledgeSearchResults, packKnowledgeSearchContext, searchKnowledge };
|
|
49
49
|
//# sourceMappingURL=search.d.mts.map
|
package/dist/search.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.mts","names":[],"sources":["../src/search.ts"],"mappings":";;;;;;;KAMY,oBAAA;EAAA,SACD,KAAA,EAAO,oBAAA;EAAA,SACP,KAAA;EAAA,SACA,KAAA;EAAA,SACA,QAAA;EAAA,SACA,aAAA;EAAA,SACA,IAAA,GAAO,mBAAmB;EAAA,SAC1B,WAAA;EAAA,SACA,SAAA;AAAA;AAAA,KAGC,mBAAA;AAAA,KAEA,qBAAA;EAAA,SACD,MAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA;AAAA;AAAA,KAGC,qBAAA;EAAA,SACD,KAAA,EAAO,cAAA;EAAA,SACP,KAAA;EAAA,SACA,QAAA,EAAU,wBAAA;EAAA,SACV,OAAA,GAAU,aAAA,CAAc,cAAA;EAAA,SACxB,MAAA,GAAS,qBAAA;AAAA;AAAA,KAGR,sBAAA;EAAA,SACD,KAAA;EAAA,SACA,OAAA,EAAS,aAAa,CAAC,qBAAA;EAAA,SACvB,IAAA;AAAA;AAAA,KAGC,iBAAA;EAAA,SACD,MAAA,GACP,KAAA,EAAO,oBAAA,KACJ,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,qBAAA,GAAwB,oBAAA;AAAA;AAAA,cAM9C,0BAAA,GACX,KAAA,UACA,OAAA,EAAS,aAAA,CAAc,qBAAA,MACtB,sBAAA;AAAA,
|
|
1
|
+
{"version":3,"file":"search.d.mts","names":[],"sources":["../src/search.ts"],"mappings":";;;;;;;KAMY,oBAAA;EAAA,SACD,KAAA,EAAO,oBAAA;EAAA,SACP,KAAA;EAAA,SACA,KAAA;EAAA,SACA,QAAA;EAAA,SACA,aAAA;EAAA,SACA,IAAA,GAAO,mBAAmB;EAAA,SAC1B,WAAA;EAAA,SACA,SAAA;AAAA;AAAA,KAGC,mBAAA;AAAA,KAEA,qBAAA;EAAA,SACD,MAAA;EAAA,SACA,IAAA;EAAA,SACA,KAAA;AAAA;AAAA,KAGC,qBAAA;EAAA,SACD,KAAA,EAAO,cAAA;EAAA,SACP,KAAA;EAAA,SACA,QAAA,EAAU,wBAAA;EAAA,SACV,OAAA,GAAU,aAAA,CAAc,cAAA;EAAA,SACxB,MAAA,GAAS,qBAAA;AAAA;AAAA,KAGR,sBAAA;EAAA,SACD,KAAA;EAAA,SACA,OAAA,EAAS,aAAa,CAAC,qBAAA;EAAA,SACvB,IAAA;AAAA;AAAA,KAGC,iBAAA;EAAA,SACD,MAAA,GACP,KAAA,EAAO,oBAAA,KACJ,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,qBAAA,GAAwB,oBAAA;AAAA;AAAA,cAM9C,0BAAA,GACX,KAAA,UACA,OAAA,EAAS,aAAA,CAAc,qBAAA,MACtB,sBAAA;AAAA,cAkBU,0BAAA,GAA8B,KAAA;EAAA,SAChC,aAAA,EAAe,aAAA,CAAc,qBAAA;EAAA,SAC7B,WAAA,EAAa,aAAA,CAAc,qBAAA;EAAA,SAC3B,KAAA;EAAA,SACA,YAAA;AAAA,MACP,aAAA,CAAc,qBAAA;AAAA,cA8JL,eAAA,GAAmB,KAAA,EAAO,oBAAA,KAAoB,MAAA,CAAA,MAAA,UAAA,qBAAA,IAAA,oBAAA,EAAA,iBAAA,GAAA,gBAAA"}
|
package/dist/search.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { KnowledgeSearchError } from "./errors.mjs";
|
|
2
2
|
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
3
3
|
import { SearchIndexStore } from "./store.mjs";
|
|
4
|
-
import { Effect } from "effect";
|
|
4
|
+
import { Effect, Match } from "effect";
|
|
5
5
|
//#region src/search.ts
|
|
6
6
|
const packedResultText = (result) => result.context?.map((chunk) => chunk.content).join("\n\n") ?? result.chunk.content;
|
|
7
7
|
const packKnowledgeSearchContext = (query, results) => ({
|
|
@@ -9,12 +9,10 @@ const packKnowledgeSearchContext = (query, results) => ({
|
|
|
9
9
|
results,
|
|
10
10
|
text: results.map(packedResultText).join("\n\n")
|
|
11
11
|
});
|
|
12
|
-
const scopeIds = (scope) => {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
17
|
-
};
|
|
12
|
+
const scopeIds = (scope) => Match.value(scope).pipe(Match.tagsExhaustive({
|
|
13
|
+
KnowledgeScope: ({ id }) => [id],
|
|
14
|
+
KnowledgeScopes: ({ ids }) => ids
|
|
15
|
+
}));
|
|
18
16
|
const defaultHybridCandidateLimit = (limit) => Math.max(limit * 5, 40);
|
|
19
17
|
const reciprocalRankFusionScore = (rank, rankConstant) => 1 / (rankConstant + rank);
|
|
20
18
|
const fuseKnowledgeSearchResults = (input) => {
|
package/dist/search.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.mjs","names":[],"sources":["../src/search.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport type { IndexedKnowledgeDocument, KnowledgeChunk, KnowledgeSearchScope } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeSearchError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type KnowledgeSearchInput = {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n readonly mode?: KnowledgeSearchMode\n readonly vectorLimit?: number\n readonly textLimit?: number\n}\n\nexport type KnowledgeSearchMode = 'vector' | 'hybrid'\n\nexport type KnowledgeSearchScores = {\n readonly vector?: number\n readonly text?: number\n readonly fused?: number\n}\n\nexport type KnowledgeSearchResult = {\n readonly chunk: KnowledgeChunk\n readonly score: number\n readonly document: IndexedKnowledgeDocument\n readonly context?: ReadonlyArray<KnowledgeChunk>\n readonly scores?: KnowledgeSearchScores\n}\n\nexport type KnowledgeSearchContext = {\n readonly query: string\n readonly results: ReadonlyArray<KnowledgeSearchResult>\n readonly text: string\n}\n\nexport type KnowledgeSearcher = {\n readonly search: (\n input: KnowledgeSearchInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeSearchResult>, KnowledgeSearchError>\n}\n\nconst packedResultText = (result: KnowledgeSearchResult) =>\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.chunk.content\n\nexport const packKnowledgeSearchContext = (\n query: string,\n results: ReadonlyArray<KnowledgeSearchResult>\n): KnowledgeSearchContext => ({\n query,\n results,\n text: results.map(packedResultText).join('\\n\\n')\n})\n\nconst scopeIds = (scope: KnowledgeSearchScope): ReadonlyArray<string> => {\n switch (scope._tag) {\n case 'KnowledgeScope':\n return [scope.id]\n case 'KnowledgeScopes':\n return scope.ids\n }\n}\n\nconst defaultHybridCandidateLimit = (limit: number) => Math.max(limit * 5, 40)\n\nconst reciprocalRankFusionScore = (rank: number, rankConstant: number) => 1 / (rankConstant + rank)\n\nexport const fuseKnowledgeSearchResults = (input: {\n readonly vectorResults: ReadonlyArray<KnowledgeSearchResult>\n readonly textResults: ReadonlyArray<KnowledgeSearchResult>\n readonly limit: number\n readonly rankConstant?: number\n}): ReadonlyArray<KnowledgeSearchResult> => {\n const rankConstant = input.rankConstant ?? 60\n const fused = new Map<\n string,\n {\n readonly result: KnowledgeSearchResult\n readonly vector?: number\n readonly text?: number\n readonly fused: number\n }\n >()\n\n const addResults = (results: ReadonlyArray<KnowledgeSearchResult>, kind: 'vector' | 'text') => {\n results.forEach((result, index) => {\n const rank = index + 1\n const previous = fused.get(result.chunk.id)\n const nextFused = (previous?.fused ?? 0) + reciprocalRankFusionScore(rank, rankConstant)\n fused.set(result.chunk.id, {\n result: previous?.result ?? result,\n vector: kind === 'vector' ? result.score : previous?.vector,\n text: kind === 'text' ? result.score : previous?.text,\n fused: nextFused\n })\n })\n }\n\n addResults(input.vectorResults, 'vector')\n addResults(input.textResults, 'text')\n\n return Array.from(fused.values())\n .sort((left, right) => right.fused - left.fused)\n .slice(0, input.limit)\n .map(item => ({\n ...item.result,\n score: item.fused,\n scores: { vector: item.vector, text: item.text, fused: item.fused }\n }))\n}\n\nconst validateSearchInput = (input: KnowledgeSearchInput) => {\n const query = input.query.trim()\n const ids = scopeIds(input.scope)\n const limit = input.limit ?? 10\n const contextChunks = input.contextChunks ?? 0\n const mode = input.mode ?? 'hybrid'\n const vectorLimit =\n input.vectorLimit ?? (mode === 'hybrid' ? defaultHybridCandidateLimit(limit) : limit)\n const textLimit = input.textLimit ?? defaultHybridCandidateLimit(limit)\n\n if (query.length === 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search query is empty', stage: 'embed' })\n )\n }\n\n if (ids.length === 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search scope is empty', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(limit) || limit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search limit must be positive', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(vectorLimit) || vectorLimit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Vector search limit must be positive', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(textLimit) || textLimit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Text search limit must be positive', stage: 'store' })\n )\n }\n\n if (mode !== 'vector' && mode !== 'hybrid') {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search mode is invalid', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(contextChunks) || contextChunks < 0) {\n return Effect.fail(\n new KnowledgeSearchError({\n message: 'Context chunks must be zero or positive',\n stage: 'store'\n })\n )\n }\n\n if (input.minScore !== undefined && !Number.isFinite(input.minScore)) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search minScore must be finite', stage: 'store' })\n )\n }\n\n return Effect.succeed({ query, limit, contextChunks, mode, vectorLimit, textLimit })\n}\n\nconst searchVectorChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n readonly minScore?: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n const embedder = yield* KnowledgeEmbedder\n const embedding = yield* embedder\n .embedQuery(input.query)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'embed', cause: error })\n )\n )\n\n return yield* store\n .searchChunks({\n scope: input.scope,\n embedding,\n limit: input.limit,\n minScore: input.minScore\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nconst searchTextChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n return yield* store\n .searchChunksByText({ scope: input.scope, query: input.query, limit: input.limit })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nexport const searchKnowledge = (input: KnowledgeSearchInput) =>\n Effect.gen(function* () {\n const valid = yield* validateSearchInput(input)\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.query_length': valid.query.length,\n 'knowledge_search.limit': valid.limit,\n 'knowledge_search.mode': valid.mode,\n 'knowledge_search.context_chunks': valid.contextChunks,\n 'knowledge_search.scope_count': scopeIds(input.scope).length\n })\n const store = yield* SearchIndexStore\n const results: ReadonlyArray<KnowledgeSearchResult> =\n valid.mode === 'vector'\n ? yield* searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n })\n : yield* Effect.gen(function* () {\n const searches = yield* Effect.all(\n {\n vectorResults: searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n }),\n textResults: searchTextChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.textLimit\n })\n },\n { concurrency: 'unbounded' }\n )\n\n return fuseKnowledgeSearchResults({\n vectorResults: searches.vectorResults,\n textResults: searches.textResults,\n limit: valid.limit\n })\n })\n\n if (valid.contextChunks === 0) {\n return results\n }\n\n return yield* Effect.forEach(results, result =>\n store\n .getContextChunks({\n scopeId: result.chunk.scopeId,\n documentId: result.chunk.documentId,\n position: result.chunk.position,\n contextChunks: valid.contextChunks\n })\n .pipe(\n Effect.map(context => ({ ...result, context })),\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n )\n }).pipe(Effect.withSpan('knowledge_search.search'))\n"],"mappings":";;;;;AA6CA,MAAM,oBAAoB,WACxB,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,MAAM;AAE3E,MAAa,8BACX,OACA,aAC4B;CAC5B;CACA;CACA,MAAM,QAAQ,IAAI,gBAAgB,EAAE,KAAK,MAAM;AACjD;AAEA,MAAM,YAAY,UAAuD;CACvE,QAAQ,MAAM,MAAd;EACE,KAAK,kBACH,OAAO,CAAC,MAAM,EAAE;EAClB,KAAK,mBACH,OAAO,MAAM;CACjB;AACF;AAEA,MAAM,+BAA+B,UAAkB,KAAK,IAAI,QAAQ,GAAG,EAAE;AAE7E,MAAM,6BAA6B,MAAc,iBAAyB,KAAK,eAAe;AAE9F,MAAa,8BAA8B,UAKC;CAC1C,MAAM,eAAe,MAAM,gBAAgB;CAC3C,MAAM,wBAAQ,IAAI,IAQhB;CAEF,MAAM,cAAc,SAA+C,SAA4B;EAC7F,QAAQ,SAAS,QAAQ,UAAU;GACjC,MAAM,OAAO,QAAQ;GACrB,MAAM,WAAW,MAAM,IAAI,OAAO,MAAM,EAAE;GAC1C,MAAM,aAAa,UAAU,SAAS,KAAK,0BAA0B,MAAM,YAAY;GACvF,MAAM,IAAI,OAAO,MAAM,IAAI;IACzB,QAAQ,UAAU,UAAU;IAC5B,QAAQ,SAAS,WAAW,OAAO,QAAQ,UAAU;IACrD,MAAM,SAAS,SAAS,OAAO,QAAQ,UAAU;IACjD,OAAO;GACT,CAAC;EACH,CAAC;CACH;CAEA,WAAW,MAAM,eAAe,QAAQ;CACxC,WAAW,MAAM,aAAa,MAAM;CAEpC,OAAO,MAAM,KAAK,MAAM,OAAO,CAAC,EAC7B,MAAM,MAAM,UAAU,MAAM,QAAQ,KAAK,KAAK,EAC9C,MAAM,GAAG,MAAM,KAAK,EACpB,KAAI,UAAS;EACZ,GAAG,KAAK;EACR,OAAO,KAAK;EACZ,QAAQ;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM,OAAO,KAAK;EAAM;CACpE,EAAE;AACN;AAEA,MAAM,uBAAuB,UAAgC;CAC3D,MAAM,QAAQ,MAAM,MAAM,KAAK;CAC/B,MAAM,MAAM,SAAS,MAAM,KAAK;CAChC,MAAM,QAAQ,MAAM,SAAS;CAC7B,MAAM,gBAAgB,MAAM,iBAAiB;CAC7C,MAAM,OAAO,MAAM,QAAQ;CAC3B,MAAM,cACJ,MAAM,gBAAgB,SAAS,WAAW,4BAA4B,KAAK,IAAI;CACjF,MAAM,YAAY,MAAM,aAAa,4BAA4B,KAAK;CAEtE,IAAI,MAAM,WAAW,GACnB,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAC/E;CAGF,IAAI,IAAI,WAAW,GACjB,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAC/E;CAGF,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GACtC,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAiC,OAAO;CAAQ,CAAC,CACvF;CAGF,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,GAClD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAwC,OAAO;CAAQ,CAAC,CAC9F;CAGF,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAsC,OAAO;CAAQ,CAAC,CAC5F;CAGF,IAAI,SAAS,YAAY,SAAS,UAChC,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAA0B,OAAO;CAAQ,CAAC,CAChF;CAGF,IAAI,CAAC,OAAO,UAAU,aAAa,KAAK,gBAAgB,GACtD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EACvB,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,IAAI,MAAM,aAAa,KAAA,KAAa,CAAC,OAAO,SAAS,MAAM,QAAQ,GACjE,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAkC,OAAO;CAAQ,CAAC,CACxF;CAGF,OAAO,OAAO,QAAQ;EAAE;EAAO;EAAO;EAAe;EAAM;EAAa;CAAU,CAAC;AACrF;AAEA,MAAM,sBAAsB,UAM1B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CAErB,MAAM,YAAY,QAAO,OADD,mBAErB,WAAW,MAAM,KAAK,EACtB,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;CAEF,OAAO,OAAO,MACX,aAAa;EACZ,OAAO,MAAM;EACb;EACA,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;AACJ,CAAC;AAEH,MAAM,oBAAoB,UAKxB,OAAO,IAAI,aAAa;CAEtB,OAAO,QAAO,OADO,kBAElB,mBAAmB;EAAE,OAAO,MAAM;EAAO,OAAO,MAAM;EAAO,OAAO,MAAM;CAAM,CAAC,EACjF,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;AACJ,CAAC;AAEH,MAAa,mBAAmB,UAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,oBAAoB,KAAK;CAC9C,OAAO,OAAO,oBAAoB;EAChC,iCAAiC,MAAM,MAAM;EAC7C,0BAA0B,MAAM;EAChC,yBAAyB,MAAM;EAC/B,mCAAmC,MAAM;EACzC,gCAAgC,SAAS,MAAM,KAAK,EAAE;CACxD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,UACJ,MAAM,SAAS,WACX,OAAO,mBAAmB;EACxB,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,IACD,OAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,WAAW,OAAO,OAAO,IAC7B;GACE,eAAe,mBAAmB;IAChC,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,UAAU,MAAM;GAClB,CAAC;GACD,aAAa,iBAAiB;IAC5B,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;GACf,CAAC;EACH,GACA,EAAE,aAAa,YAAY,CAC7B;EAEA,OAAO,2BAA2B;GAChC,eAAe,SAAS;GACxB,aAAa,SAAS;GACtB,OAAO,MAAM;EACf,CAAC;CACH,CAAC;CAEP,IAAI,MAAM,kBAAkB,GAC1B,OAAO;CAGT,OAAO,OAAO,OAAO,QAAQ,UAAS,WACpC,MACG,iBAAiB;EAChB,SAAS,OAAO,MAAM;EACtB,YAAY,OAAO,MAAM;EACzB,UAAU,OAAO,MAAM;EACvB,eAAe,MAAM;CACvB,CAAC,EACA,KACC,OAAO,KAAI,aAAY;EAAE,GAAG;EAAQ;CAAQ,EAAE,GAC9C,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF,CACJ;AACF,CAAC,EAAE,KAAK,OAAO,SAAS,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"search.mjs","names":[],"sources":["../src/search.ts"],"sourcesContent":["import { Effect, Match } from 'effect'\nimport type { IndexedKnowledgeDocument, KnowledgeChunk, KnowledgeSearchScope } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeSearchError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type KnowledgeSearchInput = {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n readonly mode?: KnowledgeSearchMode\n readonly vectorLimit?: number\n readonly textLimit?: number\n}\n\nexport type KnowledgeSearchMode = 'vector' | 'hybrid'\n\nexport type KnowledgeSearchScores = {\n readonly vector?: number\n readonly text?: number\n readonly fused?: number\n}\n\nexport type KnowledgeSearchResult = {\n readonly chunk: KnowledgeChunk\n readonly score: number\n readonly document: IndexedKnowledgeDocument\n readonly context?: ReadonlyArray<KnowledgeChunk>\n readonly scores?: KnowledgeSearchScores\n}\n\nexport type KnowledgeSearchContext = {\n readonly query: string\n readonly results: ReadonlyArray<KnowledgeSearchResult>\n readonly text: string\n}\n\nexport type KnowledgeSearcher = {\n readonly search: (\n input: KnowledgeSearchInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeSearchResult>, KnowledgeSearchError>\n}\n\nconst packedResultText = (result: KnowledgeSearchResult) =>\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.chunk.content\n\nexport const packKnowledgeSearchContext = (\n query: string,\n results: ReadonlyArray<KnowledgeSearchResult>\n): KnowledgeSearchContext => ({\n query,\n results,\n text: results.map(packedResultText).join('\\n\\n')\n})\n\nconst scopeIds = (scope: KnowledgeSearchScope): ReadonlyArray<string> =>\n Match.value(scope).pipe(\n Match.tagsExhaustive({\n KnowledgeScope: ({ id }) => [id],\n KnowledgeScopes: ({ ids }) => ids\n })\n )\n\nconst defaultHybridCandidateLimit = (limit: number) => Math.max(limit * 5, 40)\n\nconst reciprocalRankFusionScore = (rank: number, rankConstant: number) => 1 / (rankConstant + rank)\n\nexport const fuseKnowledgeSearchResults = (input: {\n readonly vectorResults: ReadonlyArray<KnowledgeSearchResult>\n readonly textResults: ReadonlyArray<KnowledgeSearchResult>\n readonly limit: number\n readonly rankConstant?: number\n}): ReadonlyArray<KnowledgeSearchResult> => {\n const rankConstant = input.rankConstant ?? 60\n\n const fused = new Map<\n string,\n {\n readonly result: KnowledgeSearchResult\n readonly vector?: number\n readonly text?: number\n readonly fused: number\n }\n >()\n\n const addResults = (results: ReadonlyArray<KnowledgeSearchResult>, kind: 'vector' | 'text') => {\n results.forEach((result, index) => {\n const rank = index + 1\n const previous = fused.get(result.chunk.id)\n const nextFused = (previous?.fused ?? 0) + reciprocalRankFusionScore(rank, rankConstant)\n fused.set(result.chunk.id, {\n result: previous?.result ?? result,\n vector: kind === 'vector' ? result.score : previous?.vector,\n text: kind === 'text' ? result.score : previous?.text,\n fused: nextFused\n })\n })\n }\n\n addResults(input.vectorResults, 'vector')\n addResults(input.textResults, 'text')\n\n return Array.from(fused.values())\n .sort((left, right) => right.fused - left.fused)\n .slice(0, input.limit)\n .map(item => ({\n ...item.result,\n score: item.fused,\n scores: { vector: item.vector, text: item.text, fused: item.fused }\n }))\n}\n\nconst validateSearchInput = (input: KnowledgeSearchInput) => {\n const query = input.query.trim()\n const ids = scopeIds(input.scope)\n const limit = input.limit ?? 10\n const contextChunks = input.contextChunks ?? 0\n const mode = input.mode ?? 'hybrid'\n\n const vectorLimit =\n input.vectorLimit ?? (mode === 'hybrid' ? defaultHybridCandidateLimit(limit) : limit)\n\n const textLimit = input.textLimit ?? defaultHybridCandidateLimit(limit)\n\n if (query.length === 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search query is empty', stage: 'embed' })\n )\n }\n\n if (ids.length === 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search scope is empty', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(limit) || limit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search limit must be positive', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(vectorLimit) || vectorLimit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Vector search limit must be positive', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(textLimit) || textLimit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Text search limit must be positive', stage: 'store' })\n )\n }\n\n if (mode !== 'vector' && mode !== 'hybrid') {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search mode is invalid', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(contextChunks) || contextChunks < 0) {\n return Effect.fail(\n new KnowledgeSearchError({\n message: 'Context chunks must be zero or positive',\n stage: 'store'\n })\n )\n }\n\n if (input.minScore !== undefined && !Number.isFinite(input.minScore)) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search minScore must be finite', stage: 'store' })\n )\n }\n\n return Effect.succeed({ query, limit, contextChunks, mode, vectorLimit, textLimit })\n}\n\nconst searchVectorChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n readonly minScore?: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n const embedder = yield* KnowledgeEmbedder\n\n const embedding = yield* embedder\n .embedQuery(input.query)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'embed', cause: error })\n )\n )\n\n return yield* store\n .searchChunks({\n scope: input.scope,\n embedding,\n limit: input.limit,\n minScore: input.minScore\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nconst searchTextChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n\n return yield* store\n .searchChunksByText({ scope: input.scope, query: input.query, limit: input.limit })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nexport const searchKnowledge = (input: KnowledgeSearchInput) =>\n Effect.gen(function* () {\n const valid = yield* validateSearchInput(input)\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.query_length': valid.query.length,\n 'knowledge_search.limit': valid.limit,\n 'knowledge_search.mode': valid.mode,\n 'knowledge_search.context_chunks': valid.contextChunks,\n 'knowledge_search.scope_count': scopeIds(input.scope).length\n })\n const store = yield* SearchIndexStore\n\n const results: ReadonlyArray<KnowledgeSearchResult> =\n valid.mode === 'vector'\n ? yield* searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n })\n : yield* Effect.gen(function* () {\n const searches = yield* Effect.all(\n {\n vectorResults: searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n }),\n textResults: searchTextChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.textLimit\n })\n },\n { concurrency: 'unbounded' }\n )\n\n return fuseKnowledgeSearchResults({\n vectorResults: searches.vectorResults,\n textResults: searches.textResults,\n limit: valid.limit\n })\n })\n\n if (valid.contextChunks === 0) {\n return results\n }\n\n return yield* Effect.forEach(results, result =>\n store\n .getContextChunks({\n scopeId: result.chunk.scopeId,\n documentId: result.chunk.documentId,\n position: result.chunk.position,\n contextChunks: valid.contextChunks\n })\n .pipe(\n Effect.map(context => ({ ...result, context })),\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n )\n }).pipe(Effect.withSpan('knowledge_search.search'))\n"],"mappings":";;;;;AA6CA,MAAM,oBAAoB,WACxB,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,MAAM;AAE3E,MAAa,8BACX,OACA,aAC4B;CAC5B;CACA;CACA,MAAM,QAAQ,IAAI,gBAAgB,EAAE,KAAK,MAAM;AACjD;AAEA,MAAM,YAAY,UAChB,MAAM,MAAM,KAAK,EAAE,KACjB,MAAM,eAAe;CACnB,iBAAiB,EAAE,SAAS,CAAC,EAAE;CAC/B,kBAAkB,EAAE,UAAU;AAChC,CAAC,CACH;AAEF,MAAM,+BAA+B,UAAkB,KAAK,IAAI,QAAQ,GAAG,EAAE;AAE7E,MAAM,6BAA6B,MAAc,iBAAyB,KAAK,eAAe;AAE9F,MAAa,8BAA8B,UAKC;CAC1C,MAAM,eAAe,MAAM,gBAAgB;CAE3C,MAAM,wBAAQ,IAAI,IAQhB;CAEF,MAAM,cAAc,SAA+C,SAA4B;EAC7F,QAAQ,SAAS,QAAQ,UAAU;GACjC,MAAM,OAAO,QAAQ;GACrB,MAAM,WAAW,MAAM,IAAI,OAAO,MAAM,EAAE;GAC1C,MAAM,aAAa,UAAU,SAAS,KAAK,0BAA0B,MAAM,YAAY;GACvF,MAAM,IAAI,OAAO,MAAM,IAAI;IACzB,QAAQ,UAAU,UAAU;IAC5B,QAAQ,SAAS,WAAW,OAAO,QAAQ,UAAU;IACrD,MAAM,SAAS,SAAS,OAAO,QAAQ,UAAU;IACjD,OAAO;GACT,CAAC;EACH,CAAC;CACH;CAEA,WAAW,MAAM,eAAe,QAAQ;CACxC,WAAW,MAAM,aAAa,MAAM;CAEpC,OAAO,MAAM,KAAK,MAAM,OAAO,CAAC,EAC7B,MAAM,MAAM,UAAU,MAAM,QAAQ,KAAK,KAAK,EAC9C,MAAM,GAAG,MAAM,KAAK,EACpB,KAAI,UAAS;EACZ,GAAG,KAAK;EACR,OAAO,KAAK;EACZ,QAAQ;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM,OAAO,KAAK;EAAM;CACpE,EAAE;AACN;AAEA,MAAM,uBAAuB,UAAgC;CAC3D,MAAM,QAAQ,MAAM,MAAM,KAAK;CAC/B,MAAM,MAAM,SAAS,MAAM,KAAK;CAChC,MAAM,QAAQ,MAAM,SAAS;CAC7B,MAAM,gBAAgB,MAAM,iBAAiB;CAC7C,MAAM,OAAO,MAAM,QAAQ;CAE3B,MAAM,cACJ,MAAM,gBAAgB,SAAS,WAAW,4BAA4B,KAAK,IAAI;CAEjF,MAAM,YAAY,MAAM,aAAa,4BAA4B,KAAK;CAEtE,IAAI,MAAM,WAAW,GACnB,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAC/E;CAGF,IAAI,IAAI,WAAW,GACjB,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAC/E;CAGF,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GACtC,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAiC,OAAO;CAAQ,CAAC,CACvF;CAGF,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,GAClD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAwC,OAAO;CAAQ,CAAC,CAC9F;CAGF,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAsC,OAAO;CAAQ,CAAC,CAC5F;CAGF,IAAI,SAAS,YAAY,SAAS,UAChC,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAA0B,OAAO;CAAQ,CAAC,CAChF;CAGF,IAAI,CAAC,OAAO,UAAU,aAAa,KAAK,gBAAgB,GACtD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EACvB,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,IAAI,MAAM,aAAa,KAAA,KAAa,CAAC,OAAO,SAAS,MAAM,QAAQ,GACjE,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAkC,OAAO;CAAQ,CAAC,CACxF;CAGF,OAAO,OAAO,QAAQ;EAAE;EAAO;EAAO;EAAe;EAAM;EAAa;CAAU,CAAC;AACrF;AAEA,MAAM,sBAAsB,UAM1B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CAGrB,MAAM,YAAY,QAAO,OAFD,mBAGrB,WAAW,MAAM,KAAK,EACtB,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;CAEF,OAAO,OAAO,MACX,aAAa;EACZ,OAAO,MAAM;EACb;EACA,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;AACJ,CAAC;AAEH,MAAM,oBAAoB,UAKxB,OAAO,IAAI,aAAa;CAGtB,OAAO,QAAO,OAFO,kBAGlB,mBAAmB;EAAE,OAAO,MAAM;EAAO,OAAO,MAAM;EAAO,OAAO,MAAM;CAAM,CAAC,EACjF,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;AACJ,CAAC;AAEH,MAAa,mBAAmB,UAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,oBAAoB,KAAK;CAC9C,OAAO,OAAO,oBAAoB;EAChC,iCAAiC,MAAM,MAAM;EAC7C,0BAA0B,MAAM;EAChC,yBAAyB,MAAM;EAC/B,mCAAmC,MAAM;EACzC,gCAAgC,SAAS,MAAM,KAAK,EAAE;CACxD,CAAC;CACD,MAAM,QAAQ,OAAO;CAErB,MAAM,UACJ,MAAM,SAAS,WACX,OAAO,mBAAmB;EACxB,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,IACD,OAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,WAAW,OAAO,OAAO,IAC7B;GACE,eAAe,mBAAmB;IAChC,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,UAAU,MAAM;GAClB,CAAC;GACD,aAAa,iBAAiB;IAC5B,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;GACf,CAAC;EACH,GACA,EAAE,aAAa,YAAY,CAC7B;EAEA,OAAO,2BAA2B;GAChC,eAAe,SAAS;GACxB,aAAa,SAAS;GACtB,OAAO,MAAM;EACf,CAAC;CACH,CAAC;CAEP,IAAI,MAAM,kBAAkB,GAC1B,OAAO;CAGT,OAAO,OAAO,OAAO,QAAQ,UAAS,WACpC,MACG,iBAAiB;EAChB,SAAS,OAAO,MAAM;EACtB,YAAY,OAAO,MAAM;EACzB,UAAU,OAAO,MAAM;EACvB,eAAe,MAAM;CACvB,CAAC,EACA,KACC,OAAO,KAAI,aAAY;EAAE,GAAG;EAAQ;CAAQ,EAAE,GAC9C,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF,CACJ;AACF,CAAC,EAAE,KAAK,OAAO,SAAS,yBAAyB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yolk-sdk/knowledge",
|
|
3
|
-
"version": "0.1.0-canary.
|
|
3
|
+
"version": "0.1.0-canary.78",
|
|
4
4
|
"description": "Domain-free knowledge document/source/file/context, ingestion, hybrid search, and lookup/manage tool helpers for agents.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -104,9 +104,9 @@
|
|
|
104
104
|
"provenance": true
|
|
105
105
|
},
|
|
106
106
|
"dependencies": {
|
|
107
|
-
"effect": "4.0.0-
|
|
107
|
+
"effect": "4.0.0-rc.115",
|
|
108
108
|
"gpt-tokenizer": "^3.4.0",
|
|
109
|
-
"@yolk-sdk/agent": "^0.1.0-canary.
|
|
109
|
+
"@yolk-sdk/agent": "^0.1.0-canary.78"
|
|
110
110
|
},
|
|
111
111
|
"scripts": {
|
|
112
112
|
"build": "tsdown",
|
package/src/agent.ts
CHANGED
|
@@ -13,8 +13,11 @@ import {
|
|
|
13
13
|
} from './documents.ts'
|
|
14
14
|
|
|
15
15
|
const OptionalNonEmptyTrimmedString = Schema.optional(Schema.NullOr(NonEmptyTrimmedString))
|
|
16
|
+
|
|
16
17
|
const OptionalPositiveInteger = Schema.optional(Schema.NullOr(PositiveInteger))
|
|
18
|
+
|
|
17
19
|
const OptionalNonNegativeInteger = Schema.optional(Schema.NullOr(NonNegativeInteger))
|
|
20
|
+
|
|
18
21
|
const OptionalNumber = Schema.optional(Schema.NullOr(Schema.Number))
|
|
19
22
|
|
|
20
23
|
const KnowledgeTargetParamsSchema = Schema.Struct({
|
|
@@ -64,7 +67,9 @@ const KnowledgeManageParams = Schema.Union([
|
|
|
64
67
|
])
|
|
65
68
|
|
|
66
69
|
type KnowledgeTargetParams = typeof KnowledgeTargetParamsSchema.Type
|
|
70
|
+
|
|
67
71
|
type KnowledgeLookupParams = typeof KnowledgeLookupParams.Type
|
|
72
|
+
|
|
68
73
|
type KnowledgeManageParams = typeof KnowledgeManageParams.Type
|
|
69
74
|
|
|
70
75
|
export type KnowledgeTarget = {
|
|
@@ -125,9 +130,6 @@ export type KnowledgeManageHandlers<Context> = {
|
|
|
125
130
|
}) => Effect.Effect<KnowledgeSavedDocument, ToolError>
|
|
126
131
|
}
|
|
127
132
|
|
|
128
|
-
const unknownToMessage = (error: unknown) =>
|
|
129
|
-
error instanceof Error ? error.message : String(error)
|
|
130
|
-
|
|
131
133
|
const optionalValue = <Value>(value: Value | null | undefined) => value ?? undefined
|
|
132
134
|
|
|
133
135
|
const targetFromParams = (target: KnowledgeTargetParams): KnowledgeTarget =>
|
|
@@ -190,7 +192,8 @@ export const makeKnowledgeLookupTool = <Context>(
|
|
|
190
192
|
'Look up durable knowledge. Search for semantic discovery; get when document_id or slug is known.',
|
|
191
193
|
parameters: KnowledgeLookupParams,
|
|
192
194
|
access: 'read',
|
|
193
|
-
invalidParamsMessage: error =>
|
|
195
|
+
invalidParamsMessage: error =>
|
|
196
|
+
`Invalid knowledge lookup arguments: ${error instanceof Error ? error.message : String(error)}`,
|
|
194
197
|
execute: ({ call, context, params }) =>
|
|
195
198
|
Effect.gen(function* () {
|
|
196
199
|
if (params.operation === 'search') {
|
|
@@ -210,6 +213,7 @@ export const makeKnowledgeLookupTool = <Context>(
|
|
|
210
213
|
id: optionalValue(params.id),
|
|
211
214
|
target: optionalTargetFromParams(params.target)
|
|
212
215
|
})
|
|
216
|
+
|
|
213
217
|
return ToolResult.make({ toolCallId: call.id, content: formatDocument(document) })
|
|
214
218
|
})
|
|
215
219
|
})
|
|
@@ -228,7 +232,8 @@ export const makeKnowledgeManageTool = <Context>(
|
|
|
228
232
|
'Create, replace, pin, archive, rename, or delete durable knowledge. Use only when explicitly asked.',
|
|
229
233
|
parameters: KnowledgeManageParams,
|
|
230
234
|
access: 'write',
|
|
231
|
-
invalidParamsMessage: error =>
|
|
235
|
+
invalidParamsMessage: error =>
|
|
236
|
+
`Invalid knowledge manage arguments: ${error instanceof Error ? error.message : String(error)}`,
|
|
232
237
|
execute: ({ call, context, params }) =>
|
|
233
238
|
Effect.gen(function* () {
|
|
234
239
|
switch (params.operation) {
|
|
@@ -242,38 +247,45 @@ export const makeKnowledgeManageTool = <Context>(
|
|
|
242
247
|
content: params.content,
|
|
243
248
|
availability: params.availability
|
|
244
249
|
})
|
|
250
|
+
|
|
245
251
|
return ToolResult.make({
|
|
246
252
|
toolCallId: call.id,
|
|
247
253
|
content: formatSaved('Knowledge upserted', document)
|
|
248
254
|
})
|
|
249
255
|
}
|
|
256
|
+
|
|
250
257
|
case 'set_availability': {
|
|
251
258
|
const document = yield* handlers.setAvailability({
|
|
252
259
|
context,
|
|
253
260
|
target: targetFromParams(params.target),
|
|
254
261
|
availability: params.availability
|
|
255
262
|
})
|
|
263
|
+
|
|
256
264
|
return ToolResult.make({
|
|
257
265
|
toolCallId: call.id,
|
|
258
266
|
content: formatSaved('Knowledge availability updated', document)
|
|
259
267
|
})
|
|
260
268
|
}
|
|
269
|
+
|
|
261
270
|
case 'rename_slug': {
|
|
262
271
|
const document = yield* handlers.renameSlug({
|
|
263
272
|
context,
|
|
264
273
|
target: targetFromParams(params.target),
|
|
265
274
|
nextSlug: params.nextSlug
|
|
266
275
|
})
|
|
276
|
+
|
|
267
277
|
return ToolResult.make({
|
|
268
278
|
toolCallId: call.id,
|
|
269
279
|
content: formatSaved('Knowledge slug renamed', document)
|
|
270
280
|
})
|
|
271
281
|
}
|
|
282
|
+
|
|
272
283
|
case 'delete': {
|
|
273
284
|
const document = yield* handlers.delete({
|
|
274
285
|
context,
|
|
275
286
|
target: targetFromParams(params.target)
|
|
276
287
|
})
|
|
288
|
+
|
|
277
289
|
return ToolResult.make({
|
|
278
290
|
toolCallId: call.id,
|
|
279
291
|
content: formatSaved('Knowledge deleted', document)
|
package/src/chunking.ts
CHANGED
|
@@ -4,7 +4,9 @@ import type { KnowledgeChunk, KnowledgeMetadata } from './documents.ts'
|
|
|
4
4
|
import { KnowledgeChunkingError } from './errors.ts'
|
|
5
5
|
|
|
6
6
|
const SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:["')\]]+)?\s*|[^.!?]+$/g
|
|
7
|
+
|
|
7
8
|
const PARAGRAPH_BREAK_PATTERN = /(\n{2,})/
|
|
9
|
+
|
|
8
10
|
const WORD_PATTERN = /\S+\s*/g
|
|
9
11
|
|
|
10
12
|
export type ChunkKnowledgeDocumentInput = {
|
|
@@ -52,6 +54,7 @@ const splitSentences = (text: string) =>
|
|
|
52
54
|
|
|
53
55
|
const splitEncodedTokens = (text: string, maxTokens: number) => {
|
|
54
56
|
const tokens = encode(text)
|
|
57
|
+
|
|
55
58
|
return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>
|
|
56
59
|
decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()
|
|
57
60
|
).filter(chunk => chunk.length > 0)
|
|
@@ -68,6 +71,7 @@ const splitOversizedUnit = (text: string, maxTokens: number) => {
|
|
|
68
71
|
|
|
69
72
|
if (wordTokenCount > maxTokens) {
|
|
70
73
|
const currentChunk = current.trim()
|
|
74
|
+
|
|
71
75
|
if (currentChunk.length > 0) {
|
|
72
76
|
chunks.push(currentChunk)
|
|
73
77
|
}
|
|
@@ -80,6 +84,7 @@ const splitOversizedUnit = (text: string, maxTokens: number) => {
|
|
|
80
84
|
|
|
81
85
|
if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {
|
|
82
86
|
const currentChunk = current.trim()
|
|
87
|
+
|
|
83
88
|
if (currentChunk.length > 0) {
|
|
84
89
|
chunks.push(currentChunk)
|
|
85
90
|
}
|
|
@@ -94,6 +99,7 @@ const splitOversizedUnit = (text: string, maxTokens: number) => {
|
|
|
94
99
|
}
|
|
95
100
|
|
|
96
101
|
const finalChunk = current.trim()
|
|
102
|
+
|
|
97
103
|
if (finalChunk.length > 0) {
|
|
98
104
|
chunks.push(finalChunk)
|
|
99
105
|
}
|
|
@@ -113,6 +119,7 @@ const buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) =>
|
|
|
113
119
|
|
|
114
120
|
const pushCurrent = () => {
|
|
115
121
|
const content = current.trim()
|
|
122
|
+
|
|
116
123
|
if (content.length > 0) {
|
|
117
124
|
chunks.push(content)
|
|
118
125
|
}
|
|
@@ -133,6 +140,7 @@ const buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) =>
|
|
|
133
140
|
}
|
|
134
141
|
|
|
135
142
|
pushCurrent()
|
|
143
|
+
|
|
136
144
|
return chunks
|
|
137
145
|
}
|
|
138
146
|
|
|
@@ -145,6 +153,7 @@ export const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens
|
|
|
145
153
|
}
|
|
146
154
|
|
|
147
155
|
const sanitized = sanitizeText(input.content)
|
|
156
|
+
|
|
148
157
|
if (sanitized.length === 0) {
|
|
149
158
|
return yield* Effect.fail(
|
|
150
159
|
new KnowledgeChunkingError({ message: 'Cannot chunk empty content' })
|
package/src/context.ts
CHANGED
|
@@ -30,6 +30,7 @@ export const buildKnowledgeContext = (input: BuildKnowledgeContextInput) => {
|
|
|
30
30
|
const sections = Arr.map(input.documents, documentBody).filter(
|
|
31
31
|
section => section.trim().length > 0
|
|
32
32
|
)
|
|
33
|
+
|
|
33
34
|
const header = '# Pinned knowledge\nUse this durable knowledge as high-priority context.'
|
|
34
35
|
const body = sections.join('\n\n')
|
|
35
36
|
const context = body.length === 0 ? '' : `${header}\n\n${body}`
|
package/src/documents.ts
CHANGED
|
@@ -1,22 +1,28 @@
|
|
|
1
1
|
import * as Schema from 'effect/Schema'
|
|
2
2
|
|
|
3
3
|
export const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))
|
|
4
|
+
|
|
4
5
|
export const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))
|
|
6
|
+
|
|
5
7
|
export const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))
|
|
6
8
|
|
|
7
9
|
export const KnowledgeMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)
|
|
10
|
+
|
|
8
11
|
export type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>
|
|
9
12
|
|
|
10
13
|
export const KnowledgeAvailabilitySchema = Schema.Literals(['pinned', 'searchable', 'archived'])
|
|
14
|
+
|
|
11
15
|
export type KnowledgeAvailability = Schema.Schema.Type<typeof KnowledgeAvailabilitySchema>
|
|
12
16
|
|
|
13
17
|
export const KnowledgeDocumentStatusSchema = Schema.Literals(['processing', 'ready', 'error'])
|
|
18
|
+
|
|
14
19
|
export type KnowledgeDocumentStatus = Schema.Schema.Type<typeof KnowledgeDocumentStatusSchema>
|
|
15
20
|
|
|
16
21
|
export const KnowledgeScopeSchema = Schema.Struct({
|
|
17
22
|
id: NonEmptyTrimmedString,
|
|
18
23
|
kind: Schema.optional(NonEmptyTrimmedString)
|
|
19
24
|
})
|
|
25
|
+
|
|
20
26
|
export type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>
|
|
21
27
|
|
|
22
28
|
export const KnowledgeDocumentSchema = Schema.Struct({
|
|
@@ -35,6 +41,7 @@ export const KnowledgeDocumentSchema = Schema.Struct({
|
|
|
35
41
|
createdAt: Schema.DateTimeUtc,
|
|
36
42
|
updatedAt: Schema.DateTimeUtc
|
|
37
43
|
})
|
|
44
|
+
|
|
38
45
|
export type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>
|
|
39
46
|
|
|
40
47
|
export const CreateKnowledgeDocumentInputSchema = Schema.Struct({
|
|
@@ -48,6 +55,7 @@ export const CreateKnowledgeDocumentInputSchema = Schema.Struct({
|
|
|
48
55
|
summary: Schema.optional(Schema.String),
|
|
49
56
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
50
57
|
})
|
|
58
|
+
|
|
51
59
|
export type CreateKnowledgeDocumentInput = Schema.Schema.Type<
|
|
52
60
|
typeof CreateKnowledgeDocumentInputSchema
|
|
53
61
|
>
|
|
@@ -67,6 +75,7 @@ export const UpdateKnowledgeDocumentInputSchema = Schema.Struct({
|
|
|
67
75
|
reviewedAt: Schema.optional(Schema.DateTimeUtc),
|
|
68
76
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
69
77
|
})
|
|
78
|
+
|
|
70
79
|
export type UpdateKnowledgeDocumentInput = Schema.Schema.Type<
|
|
71
80
|
typeof UpdateKnowledgeDocumentInputSchema
|
|
72
81
|
>
|
|
@@ -81,6 +90,7 @@ export const KnowledgeFileSchema = Schema.Struct({
|
|
|
81
90
|
metadata: Schema.optional(KnowledgeMetadataSchema),
|
|
82
91
|
createdAt: Schema.DateTimeUtc
|
|
83
92
|
})
|
|
93
|
+
|
|
84
94
|
export type KnowledgeFile = Schema.Schema.Type<typeof KnowledgeFileSchema>
|
|
85
95
|
|
|
86
96
|
export const KnowledgeChunkSchema = Schema.Struct({
|
|
@@ -92,34 +102,45 @@ export const KnowledgeChunkSchema = Schema.Struct({
|
|
|
92
102
|
tokenCount: PositiveInteger,
|
|
93
103
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
94
104
|
})
|
|
105
|
+
|
|
95
106
|
export type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>
|
|
96
107
|
|
|
108
|
+
export const KnowledgeSearchScope = Schema.TaggedStruct('KnowledgeScope', {
|
|
109
|
+
id: NonEmptyTrimmedString
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
export const KnowledgeSearchScopes = Schema.TaggedStruct('KnowledgeScopes', {
|
|
113
|
+
ids: Schema.NonEmptyArray(NonEmptyTrimmedString)
|
|
114
|
+
})
|
|
115
|
+
|
|
97
116
|
export const KnowledgeSearchScopeSchema = Schema.Union([
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
_tag: Schema.Literal('KnowledgeScopes'),
|
|
101
|
-
ids: Schema.NonEmptyArray(NonEmptyTrimmedString)
|
|
102
|
-
})
|
|
117
|
+
KnowledgeSearchScope,
|
|
118
|
+
KnowledgeSearchScopes
|
|
103
119
|
])
|
|
104
|
-
|
|
120
|
+
|
|
121
|
+
export type KnowledgeSearchScope = typeof KnowledgeSearchScopeSchema.Type
|
|
122
|
+
|
|
123
|
+
export const KnowledgeFileSource = Schema.TaggedStruct('File', {
|
|
124
|
+
ref: NonEmptyTrimmedString,
|
|
125
|
+
name: Schema.optional(NonEmptyTrimmedString),
|
|
126
|
+
mediaType: Schema.optional(NonEmptyTrimmedString)
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
export const KnowledgeUrlSource = Schema.TaggedStruct('Url', {
|
|
130
|
+
url: NonEmptyTrimmedString
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
export const KnowledgeTextSource = Schema.TaggedStruct('Text', {
|
|
134
|
+
label: Schema.optional(NonEmptyTrimmedString)
|
|
135
|
+
})
|
|
105
136
|
|
|
106
137
|
export const KnowledgeSourceSchema = Schema.Union([
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
name: Schema.optional(NonEmptyTrimmedString),
|
|
111
|
-
mediaType: Schema.optional(NonEmptyTrimmedString)
|
|
112
|
-
}),
|
|
113
|
-
Schema.Struct({
|
|
114
|
-
_tag: Schema.Literal('Url'),
|
|
115
|
-
url: NonEmptyTrimmedString
|
|
116
|
-
}),
|
|
117
|
-
Schema.Struct({
|
|
118
|
-
_tag: Schema.Literal('Text'),
|
|
119
|
-
label: Schema.optional(NonEmptyTrimmedString)
|
|
120
|
-
})
|
|
138
|
+
KnowledgeFileSource,
|
|
139
|
+
KnowledgeUrlSource,
|
|
140
|
+
KnowledgeTextSource
|
|
121
141
|
])
|
|
122
|
-
|
|
142
|
+
|
|
143
|
+
export type KnowledgeSource = typeof KnowledgeSourceSchema.Type
|
|
123
144
|
|
|
124
145
|
export const IndexedKnowledgeDocumentSchema = Schema.Struct({
|
|
125
146
|
id: NonEmptyTrimmedString,
|
|
@@ -134,6 +155,7 @@ export const IndexedKnowledgeDocumentSchema = Schema.Struct({
|
|
|
134
155
|
chunkCount: Schema.optional(NonNegativeInteger),
|
|
135
156
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
136
157
|
})
|
|
158
|
+
|
|
137
159
|
export type IndexedKnowledgeDocument = Schema.Schema.Type<typeof IndexedKnowledgeDocumentSchema>
|
|
138
160
|
|
|
139
161
|
export const ExtractedKnowledgeDocumentSchema = Schema.Struct({
|
|
@@ -142,6 +164,7 @@ export const ExtractedKnowledgeDocumentSchema = Schema.Struct({
|
|
|
142
164
|
summary: Schema.optional(Schema.String),
|
|
143
165
|
metadata: Schema.optional(KnowledgeMetadataSchema)
|
|
144
166
|
})
|
|
167
|
+
|
|
145
168
|
export type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>
|
|
146
169
|
|
|
147
170
|
export const defaultKnowledgeChunkMaxTokens = 512
|
package/src/errors.ts
CHANGED
|
@@ -38,7 +38,7 @@ export class KnowledgeSearchError extends Data.TaggedError('KnowledgeSearchError
|
|
|
38
38
|
readonly cause?: unknown
|
|
39
39
|
}> {}
|
|
40
40
|
|
|
41
|
-
export class KnowledgeStoreError extends Schema.
|
|
41
|
+
export class KnowledgeStoreError extends Schema.TaggedError<KnowledgeStoreError>()(
|
|
42
42
|
'KnowledgeStoreError',
|
|
43
43
|
{
|
|
44
44
|
message: Schema.String,
|
|
@@ -46,7 +46,7 @@ export class KnowledgeStoreError extends Schema.TaggedErrorClass<KnowledgeStoreE
|
|
|
46
46
|
}
|
|
47
47
|
) {}
|
|
48
48
|
|
|
49
|
-
export class KnowledgeFileError extends Schema.
|
|
49
|
+
export class KnowledgeFileError extends Schema.TaggedError<KnowledgeFileError>()(
|
|
50
50
|
'KnowledgeFileError',
|
|
51
51
|
{
|
|
52
52
|
message: Schema.String,
|
|
@@ -54,7 +54,7 @@ export class KnowledgeFileError extends Schema.TaggedErrorClass<KnowledgeFileErr
|
|
|
54
54
|
}
|
|
55
55
|
) {}
|
|
56
56
|
|
|
57
|
-
export class KnowledgeContextError extends Schema.
|
|
57
|
+
export class KnowledgeContextError extends Schema.TaggedError<KnowledgeContextError>()(
|
|
58
58
|
'KnowledgeContextError',
|
|
59
59
|
{
|
|
60
60
|
message: Schema.String,
|
package/src/index.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
export { buildKnowledgeContext } from './context.ts'
|
|
2
|
+
|
|
2
3
|
export { KnowledgeStore, SearchIndexStore } from './store.ts'
|
|
4
|
+
|
|
3
5
|
export { KnowledgeFileBlobStore } from './files.ts'
|
|
6
|
+
|
|
4
7
|
export { KnowledgeStoreError, KnowledgeFileError, KnowledgeContextError } from './errors.ts'
|
package/src/ingestion.ts
CHANGED
|
@@ -125,6 +125,7 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
|
|
|
125
125
|
chunk,
|
|
126
126
|
embedding
|
|
127
127
|
}))
|
|
128
|
+
|
|
128
129
|
yield* store
|
|
129
130
|
.replaceDocumentChunks({
|
|
130
131
|
scopeId: input.scopeId,
|
|
@@ -139,6 +140,7 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
|
|
|
139
140
|
)
|
|
140
141
|
|
|
141
142
|
const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)
|
|
143
|
+
|
|
142
144
|
return yield* store
|
|
143
145
|
.markDocumentReady({
|
|
144
146
|
scopeId: input.scopeId,
|
package/src/search.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Effect } from 'effect'
|
|
1
|
+
import { Effect, Match } from 'effect'
|
|
2
2
|
import type { IndexedKnowledgeDocument, KnowledgeChunk, KnowledgeSearchScope } from './documents.ts'
|
|
3
3
|
import { KnowledgeEmbedder } from './embeddings.ts'
|
|
4
4
|
import { KnowledgeSearchError } from './errors.ts'
|
|
@@ -55,14 +55,13 @@ export const packKnowledgeSearchContext = (
|
|
|
55
55
|
text: results.map(packedResultText).join('\n\n')
|
|
56
56
|
})
|
|
57
57
|
|
|
58
|
-
const scopeIds = (scope: KnowledgeSearchScope): ReadonlyArray<string> =>
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
}
|
|
58
|
+
const scopeIds = (scope: KnowledgeSearchScope): ReadonlyArray<string> =>
|
|
59
|
+
Match.value(scope).pipe(
|
|
60
|
+
Match.tagsExhaustive({
|
|
61
|
+
KnowledgeScope: ({ id }) => [id],
|
|
62
|
+
KnowledgeScopes: ({ ids }) => ids
|
|
63
|
+
})
|
|
64
|
+
)
|
|
66
65
|
|
|
67
66
|
const defaultHybridCandidateLimit = (limit: number) => Math.max(limit * 5, 40)
|
|
68
67
|
|
|
@@ -75,6 +74,7 @@ export const fuseKnowledgeSearchResults = (input: {
|
|
|
75
74
|
readonly rankConstant?: number
|
|
76
75
|
}): ReadonlyArray<KnowledgeSearchResult> => {
|
|
77
76
|
const rankConstant = input.rankConstant ?? 60
|
|
77
|
+
|
|
78
78
|
const fused = new Map<
|
|
79
79
|
string,
|
|
80
80
|
{
|
|
@@ -118,8 +118,10 @@ const validateSearchInput = (input: KnowledgeSearchInput) => {
|
|
|
118
118
|
const limit = input.limit ?? 10
|
|
119
119
|
const contextChunks = input.contextChunks ?? 0
|
|
120
120
|
const mode = input.mode ?? 'hybrid'
|
|
121
|
+
|
|
121
122
|
const vectorLimit =
|
|
122
123
|
input.vectorLimit ?? (mode === 'hybrid' ? defaultHybridCandidateLimit(limit) : limit)
|
|
124
|
+
|
|
123
125
|
const textLimit = input.textLimit ?? defaultHybridCandidateLimit(limit)
|
|
124
126
|
|
|
125
127
|
if (query.length === 0) {
|
|
@@ -185,6 +187,7 @@ const searchVectorChunks = (input: {
|
|
|
185
187
|
Effect.gen(function* () {
|
|
186
188
|
const store = yield* SearchIndexStore
|
|
187
189
|
const embedder = yield* KnowledgeEmbedder
|
|
190
|
+
|
|
188
191
|
const embedding = yield* embedder
|
|
189
192
|
.embedQuery(input.query)
|
|
190
193
|
.pipe(
|
|
@@ -216,6 +219,7 @@ const searchTextChunks = (input: {
|
|
|
216
219
|
}) =>
|
|
217
220
|
Effect.gen(function* () {
|
|
218
221
|
const store = yield* SearchIndexStore
|
|
222
|
+
|
|
219
223
|
return yield* store
|
|
220
224
|
.searchChunksByText({ scope: input.scope, query: input.query, limit: input.limit })
|
|
221
225
|
.pipe(
|
|
@@ -237,6 +241,7 @@ export const searchKnowledge = (input: KnowledgeSearchInput) =>
|
|
|
237
241
|
'knowledge_search.scope_count': scopeIds(input.scope).length
|
|
238
242
|
})
|
|
239
243
|
const store = yield* SearchIndexStore
|
|
244
|
+
|
|
240
245
|
const results: ReadonlyArray<KnowledgeSearchResult> =
|
|
241
246
|
valid.mode === 'vector'
|
|
242
247
|
? yield* searchVectorChunks({
|