@yolk-sdk/knowledge 0.0.1-canary.0

package/dist/store.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.mts","names":[],"sources":["../src/store.ts"],"mappings":";;;;;;;;;KAcY,uBAAA;EAAA,SACD,KAAA,EAAO,cAAc;EAAA,SACrB,EAAA;AAAA;AAAA,KAGC,yBAAA;EAAA,SACD,KAAA,EAAO,cAAc;EAAA,SACrB,KAAA;AAAA;AAAA,KAGC,0BAAA;EAAA,SACD,OAAA,EAAS,aAAa,CAAC,eAAA;AAAA;AAAA,KAGtB,wBAAA;EAAA,SACD,KAAA,EAAO,cAAc;EAAA,SACrB,KAAA;AAAA;AAAA,KAGC,yBAAA;EAAA,SACD,OAAA,EAAS,aAAA,CAAc,eAAA;EAAA,SACvB,eAAA,EAAiB,aAAA,CAAc,uBAAA;AAAA;AAAA,KAG9B,iBAAA;EAAA,SACD,YAAA,GAAe,KAAA,EAAO,0BAAA,KAA+B,MAAA,CAAO,MAAA,CAAO,eAAA,EAAiB,mBAAA;EAAA,SACpF,YAAA,GAAe,KAAA,EAAO,0BAAA,KAA+B,MAAA,CAAO,MAAA,CAAO,eAAA,EAAiB,mBAAA;EAAA,SACpF,SAAA,GAAY,KAAA,EAAO,uBAAA,KAA4B,MAAA,CAAO,MAAA,CAAO,eAAA,EAAiB,mBAAA;EAAA,SAC9E,WAAA,GAAc,KAAA,EAAO,yBAAA,KAA8B,MAAA,CAAO,MAAA,CAAO,0BAAA,EAA4B,mBAAA;EAAA,SAC7F,UAAA,GAAa,KAAA,EAAO,wBAAA,KAA6B,MAAA,CAAO,MAAA,CAAO,yBAAA,EAA2B,mBAAA;EAAA,SAC1F,YAAA,GAAe,KAAA,EAAO,uBAAA,KAA4B,MAAA,CAAO,MAAA,OAAa,mBAAA;EAAA,SACtE,aAAA,GAAgB,KAAA,EAAO,uBAAA,KAA4B,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,iBAAA,GAAoB,mBAAA;EAAA,SACnG,cAAA,GAAiB,KAAA,EAAO,uBAAA,KAA4B,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,mBAAA,GAAsB,mBAAA;EAAA,SACtG,SAAA,GAAY,KAAA,EAAO,uBAAA,KAA4B,MAAA,CAAO,MAAA,CAAO,aAAA,CAAc,aAAA,GAAgB,mBAAA;AAAA;AAAA,cACrG,mBAAA;cAEY,cAAA,SAAuB,mBAEnC"}

package/dist/store.mjs

@@ -0,0 +1,7 @@
+import { Context } from "effect";
+//#region src/store.ts
+var KnowledgeStore = class extends Context.Service()("@yolk-sdk/knowledge/KnowledgeStore") {};
+//#endregion
+export { KnowledgeStore };
+
+//# sourceMappingURL=store.mjs.map

package/dist/store.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.mjs","names":[],"sources":["../src/store.ts"],"sourcesContent":["import { Context } from 'effect'\nimport type { Effect } from 'effect'\nimport type { KnowledgeArtifact } from './artifacts.ts'\nimport type { KnowledgeStoreError } from './errors.ts'\nimport type { KnowledgeLink } from './links.ts'\nimport type {\n  CreateKnowledgeRecordInput,\n  KnowledgeRecord,\n  KnowledgeScope,\n  UpdateKnowledgeRecordInput\n} from './records.ts'\nimport type { KnowledgeProvenance } from './provenance.ts'\nimport type { KnowledgeRepresentation } from './representations.ts'\n\nexport type GetKnowledgeRecordInput = {\n  readonly scope: KnowledgeScope\n  readonly id: string\n}\n\nexport type ListKnowledgeRecordsInput = {\n  readonly scope: KnowledgeScope\n  readonly limit: number\n}\n\nexport type ListKnowledgeRecordsResult = {\n  readonly records: ReadonlyArray<KnowledgeRecord>\n}\n\nexport type ListPinnedKnowledgeInput = {\n  readonly scope: KnowledgeScope\n  readonly limit: number\n}\n\nexport type ListPinnedKnowledgeResult = {\n  readonly records: ReadonlyArray<KnowledgeRecord>\n  readonly representations: ReadonlyArray<KnowledgeRepresentation>\n}\n\nexport type KnowledgeStoreApi = {\n  readonly createRecord: (input: CreateKnowledgeRecordInput) => Effect.Effect<KnowledgeRecord, KnowledgeStoreError>\n  readonly updateRecord: (input: UpdateKnowledgeRecordInput) => Effect.Effect<KnowledgeRecord, KnowledgeStoreError>\n  readonly getRecord: (input: GetKnowledgeRecordInput) => Effect.Effect<KnowledgeRecord, KnowledgeStoreError>\n  readonly listRecords: (input: ListKnowledgeRecordsInput) => Effect.Effect<ListKnowledgeRecordsResult, KnowledgeStoreError>\n  readonly listPinned: (input: ListPinnedKnowledgeInput) => Effect.Effect<ListPinnedKnowledgeResult, KnowledgeStoreError>\n  readonly deleteRecord: (input: GetKnowledgeRecordInput) => Effect.Effect<void, KnowledgeStoreError>\n  readonly listArtifacts: (input: GetKnowledgeRecordInput) => Effect.Effect<ReadonlyArray<KnowledgeArtifact>, KnowledgeStoreError>\n  readonly listProvenance: (input: GetKnowledgeRecordInput) => Effect.Effect<ReadonlyArray<KnowledgeProvenance>, KnowledgeStoreError>\n  readonly listLinks: (input: GetKnowledgeRecordInput) => Effect.Effect<ReadonlyArray<KnowledgeLink>, KnowledgeStoreError>\n}\n\nexport class KnowledgeStore extends Context.Service<KnowledgeStore, KnowledgeStoreApi>()(\n  '@yolk-sdk/knowledge/KnowledgeStore'\n) {}\n"],"mappings":";;AAkDA,IAAa,iBAAb,cAAoC,QAAQ,QAA2C,EACrF,oCACF,EAAE,CAAC"}

package/dist/summarization.d.mts

@@ -0,0 +1,23 @@
+import { KnowledgeMetadata } from "./documents.mjs";
+import { KnowledgeSummarizationError } from "./errors.mjs";
+import { Context, Effect, Layer } from "effect";
+
+//#region src/summarization.d.ts
+type KnowledgeDocumentSummary = {
+  readonly title?: string;
+  readonly summary?: string;
+};
+type SummarizeKnowledgeDocumentInput = {
+  readonly content: string;
+  readonly sourceTitle?: string;
+  readonly metadata?: KnowledgeMetadata;
+};
+type KnowledgeSummarizerApi = {
+  readonly summarize: (input: SummarizeKnowledgeDocumentInput) => Effect.Effect<KnowledgeDocumentSummary, KnowledgeSummarizationError>;
+};
+declare const KnowledgeSummarizer_base: Context.ServiceClass<KnowledgeSummarizer, "@yolk-sdk/knowledge/KnowledgeSummarizer", KnowledgeSummarizerApi>;
+declare class KnowledgeSummarizer extends KnowledgeSummarizer_base {}
+declare const NoopKnowledgeSummarizerLive: Layer.Layer<KnowledgeSummarizer, never, never>;
+//#endregion
+export { KnowledgeDocumentSummary, KnowledgeSummarizer, KnowledgeSummarizerApi, NoopKnowledgeSummarizerLive, SummarizeKnowledgeDocumentInput };
+//# sourceMappingURL=summarization.d.mts.map

package/dist/summarization.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"summarization.d.mts","names":[],"sources":["../src/summarization.ts"],"mappings":";;;;;KAIY,wBAAA;EAAA,SACD,KAAA;EAAA,SACA,OAAO;AAAA;AAAA,KAGN,+BAAA;EAAA,SACD,OAAA;EAAA,SACA,WAAA;EAAA,SACA,QAAA,GAAW,iBAAiB;AAAA;AAAA,KAG3B,sBAAA;EAAA,SACD,SAAA,GACP,KAAA,EAAO,+BAAA,KACJ,MAAA,CAAO,MAAA,CAAO,wBAAA,EAA0B,2BAAA;AAAA;AAAA,cAC9C,wBAAA;cAEY,mBAAA,SAA4B,wBAExC;AAAA,cAEY,2BAAA,EAA2B,KAAA,CAAA,KAAA,CAAA,mBAAA"}

package/dist/summarization.mjs

@@ -0,0 +1,8 @@
+import { Context, Effect, Layer } from "effect";
+//#region src/summarization.ts
+var KnowledgeSummarizer = class extends Context.Service()("@yolk-sdk/knowledge/KnowledgeSummarizer") {};
+const NoopKnowledgeSummarizerLive = Layer.succeed(KnowledgeSummarizer, { summarize: (input) => Effect.succeed({ title: input.sourceTitle }) });
+//#endregion
+export { KnowledgeSummarizer, NoopKnowledgeSummarizerLive };
+
+//# sourceMappingURL=summarization.mjs.map

package/dist/summarization.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"summarization.mjs","names":[],"sources":["../src/summarization.ts"],"sourcesContent":["import { Context, Effect, Layer } from 'effect'\nimport type { KnowledgeMetadata } from './documents.ts'\nimport type { KnowledgeSummarizationError } from './errors.ts'\n\nexport type KnowledgeDocumentSummary = {\n  readonly title?: string\n  readonly summary?: string\n}\n\nexport type SummarizeKnowledgeDocumentInput = {\n  readonly content: string\n  readonly sourceTitle?: string\n  readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeSummarizerApi = {\n  readonly summarize: (\n    input: SummarizeKnowledgeDocumentInput\n  ) => Effect.Effect<KnowledgeDocumentSummary, KnowledgeSummarizationError>\n}\n\nexport class KnowledgeSummarizer extends Context.Service<KnowledgeSummarizer, KnowledgeSummarizerApi>()(\n  '@yolk-sdk/knowledge/KnowledgeSummarizer'\n) {}\n\nexport const NoopKnowledgeSummarizerLive = Layer.succeed(KnowledgeSummarizer, {\n  summarize: input => Effect.succeed({ title: input.sourceTitle })\n})\n"],"mappings":";;AAqBA,IAAa,sBAAb,cAAyC,QAAQ,QAAqD,EACpG,yCACF,EAAE,CAAC;AAEH,MAAa,8BAA8B,MAAM,QAAQ,qBAAqB,EAC5E,YAAW,UAAS,OAAO,QAAQ,EAAE,OAAO,MAAM,YAAY,CAAC,EACjE,CAAC"}

package/dist/vector-store.d.mts

@@ -0,0 +1,3 @@
+import { SearchIndexStoreError } from "./errors.mjs";
+import { KnowledgeChunkSearchInput, KnowledgeChunkSearchResult, ReplaceKnowledgeDocumentChunksInput } from "./search-store.mjs";
+export { type ReplaceKnowledgeDocumentChunksInput as VectorRecord, type KnowledgeChunkSearchInput as VectorSearchQuery, type KnowledgeChunkSearchResult as VectorSearchResult, SearchIndexStoreError as VectorStoreError };

package/dist/vector-store.mjs

@@ -0,0 +1,2 @@
+import { SearchIndexStoreError } from "./errors.mjs";
+export { SearchIndexStoreError as VectorStoreError };

package/package.json

@@ -0,0 +1,147 @@
+{
+  "name": "@yolk-sdk/knowledge",
+  "version": "0.0.1-canary.0",
+  "description": "Domain-free knowledge record, artifact, provenance, and context contracts for Yolk agents.",
+  "license": "MIT",
+  "type": "module",
+  "sideEffects": false,
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/magoz/yolk-sdk.git",
+    "directory": "packages/knowledge"
+  },
+  "bugs": {
+    "url": "https://github.com/magoz/yolk-sdk/issues"
+  },
+  "homepage": "https://github.com/magoz/yolk-sdk#readme",
+  "keywords": [
+    "knowledge",
+    "agents",
+    "context",
+    "provenance",
+    "effect"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs",
+      "default": "./dist/index.mjs"
+    },
+    "./agent": {
+      "types": "./dist/agent.d.mts",
+      "import": "./dist/agent.mjs",
+      "default": "./dist/agent.mjs"
+    },
+    "./artifacts": {
+      "types": "./dist/artifacts.d.mts",
+      "import": "./dist/artifacts.mjs",
+      "default": "./dist/artifacts.mjs"
+    },
+    "./context": {
+      "types": "./dist/context.d.mts",
+      "import": "./dist/context.mjs",
+      "default": "./dist/context.mjs"
+    },
+    "./chunking": {
+      "types": "./dist/chunking.d.mts",
+      "import": "./dist/chunking.mjs",
+      "default": "./dist/chunking.mjs"
+    },
+    "./embeddings": {
+      "types": "./dist/embeddings.d.mts",
+      "import": "./dist/embeddings.mjs",
+      "default": "./dist/embeddings.mjs"
+    },
+    "./errors": {
+      "types": "./dist/errors.d.mts",
+      "import": "./dist/errors.mjs",
+      "default": "./dist/errors.mjs"
+    },
+    "./extraction": {
+      "types": "./dist/extraction.d.mts",
+      "import": "./dist/extraction.mjs",
+      "default": "./dist/extraction.mjs"
+    },
+    "./ingestion": {
+      "types": "./dist/ingestion.d.mts",
+      "import": "./dist/ingestion.mjs",
+      "default": "./dist/ingestion.mjs"
+    },
+    "./documents": {
+      "types": "./dist/documents.d.mts",
+      "import": "./dist/documents.mjs",
+      "default": "./dist/documents.mjs"
+    },
+    "./search-store": {
+      "types": "./dist/search-store.d.mts",
+      "import": "./dist/search-store.mjs",
+      "default": "./dist/search-store.mjs"
+    },
+    "./links": {
+      "types": "./dist/links.d.mts",
+      "import": "./dist/links.mjs",
+      "default": "./dist/links.mjs"
+    },
+    "./records": {
+      "types": "./dist/records.d.mts",
+      "import": "./dist/records.mjs",
+      "default": "./dist/records.mjs"
+    },
+    "./provenance": {
+      "types": "./dist/provenance.d.mts",
+      "import": "./dist/provenance.mjs",
+      "default": "./dist/provenance.mjs"
+    },
+    "./representations": {
+      "types": "./dist/representations.d.mts",
+      "import": "./dist/representations.mjs",
+      "default": "./dist/representations.mjs"
+    },
+    "./search": {
+      "types": "./dist/search.d.mts",
+      "import": "./dist/search.mjs",
+      "default": "./dist/search.mjs"
+    },
+    "./store": {
+      "types": "./dist/store.d.mts",
+      "import": "./dist/store.mjs",
+      "default": "./dist/store.mjs"
+    },
+    "./summarization": {
+      "types": "./dist/summarization.d.mts",
+      "import": "./dist/summarization.mjs",
+      "default": "./dist/summarization.mjs"
+    },
+    "./vector-store": {
+      "types": "./dist/vector-store.d.mts",
+      "import": "./dist/vector-store.mjs",
+      "default": "./dist/vector-store.mjs"
+    }
+  },
+  "files": [
+    "src/**/*.ts",
+    "!src/**/*.test.ts",
+    "!src/**/*.test.tsx",
+    "dist/**/*",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "dependencies": {
+    "effect": "4.0.0-beta.65",
+    "gpt-tokenizer": "^3.4.0",
+    "@yolk-sdk/agent": "^0.0.1-canary.0"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "check": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run --passWithNoTests",
+    "test:run": "vitest run --passWithNoTests"
+  }
+}

package/src/agent.ts

@@ -0,0 +1,96 @@
+import { Effect } from 'effect'
+import * as Schema from 'effect/Schema'
+import { ToolError } from '@yolk-sdk/agent/loop'
+import { ToolResult } from '@yolk-sdk/agent/protocol'
+import { makeTool, type ToolRegistration } from '@yolk-sdk/agent/tools'
+import type { KnowledgeScope } from './records.ts'
+import type { KnowledgeSearchScope } from './documents.ts'
+import type { KnowledgeSearcher } from './search.ts'
+import { packKnowledgeSearchContext } from './search.ts'
+
+export type ResolveKnowledgeScope<Context> = (context: Context) => KnowledgeScope
+
+export type KnowledgeAgentContextOptions = {
+  readonly maxPinnedContextCharacters: number
+}
+
+export const defaultKnowledgeAgentContextOptions: KnowledgeAgentContextOptions = {
+  maxPinnedContextCharacters: 6000
+}
+
+const KnowledgeSearchToolParams = Schema.Struct({
+  query: Schema.Trimmed.pipe(
+    Schema.check(Schema.isNonEmpty()),
+    Schema.annotate({ description: 'Search query for the configured knowledge search.' })
+  )
+})
+
+const isToolError = Schema.is(ToolError)
+
+export type KnowledgeSearchToolScopeResolver<Context> =
+  | KnowledgeSearchScope
+  | ((context: Context) => Effect.Effect<KnowledgeSearchScope, ToolError>)
+
+export type MakeKnowledgeSearchToolOptions<Context> = {
+  readonly scope: KnowledgeSearchToolScopeResolver<Context>
+  readonly name?: string
+  readonly description?: string
+  readonly limit?: number
+  readonly minScore?: number
+  readonly contextChunks?: number
+}
+
+const resolveScope = <Context>(resolver: KnowledgeSearchToolScopeResolver<Context>, context: Context) => {
+  if (typeof resolver === 'function') {
+    return resolver(context)
+  }
+
+  return Effect.succeed(resolver)
+}
+
+export const makeKnowledgeSearchTool = <Context>(
+  searcher: KnowledgeSearcher,
+  options: MakeKnowledgeSearchToolOptions<Context>
+): ToolRegistration<Context> => {
+  const name = options.name ?? 'search_knowledge'
+
+  return makeTool({
+    name,
+    description: options.description ?? 'Search the configured knowledge search.',
+    parameters: KnowledgeSearchToolParams,
+    access: 'read',
+    invalidParamsMessage: error => error instanceof Error ? error.message : String(error),
+    execute: input =>
+      resolveScope(options.scope, input.context).pipe(
+        Effect.flatMap(scope =>
+          searcher.search({
+            scope,
+            query: input.params.query,
+            limit: options.limit,
+            minScore: options.minScore,
+            contextChunks: options.contextChunks
+          })
+        ),
+        Effect.map(results => packKnowledgeSearchContext(input.params.query, results)),
+        Effect.map(
+          context =>
+            ToolResult.make({
+              toolCallId: input.call.id,
+              content: context.text,
+              structuredContent: context
+            })
+        ),
+        Effect.mapError(error => {
+          if (isToolError(error)) {
+            return error
+          }
+
+          return new ToolError({
+            tool: input.call.name,
+            message: error.message,
+            cause: 'execution'
+          })
+        })
+      )
+  })
+}

package/src/artifacts.ts

@@ -0,0 +1,48 @@
+import { Context } from 'effect'
+import type { Effect } from 'effect'
+import * as Schema from 'effect/Schema'
+import type { KnowledgeArtifactError } from './errors.ts'
+import { KnowledgeMetadataSchema, NonEmptyTrimmedString, NonNegativeInteger } from './records.ts'
+
+export const KnowledgeArtifactKindSchema = Schema.Literals([
+  'original',
+  'extracted_text',
+  'thumbnail',
+  'transcript',
+  'caption',
+  'structured'
+])
+export type KnowledgeArtifactKind = Schema.Schema.Type<typeof KnowledgeArtifactKindSchema>
+
+export const KnowledgeArtifactSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  recordId: NonEmptyTrimmedString,
+  kind: KnowledgeArtifactKindSchema,
+  storageKey: NonEmptyTrimmedString,
+  mediaType: Schema.optional(NonEmptyTrimmedString),
+  byteSize: Schema.optional(NonNegativeInteger),
+  checksum: Schema.optional(NonEmptyTrimmedString),
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc
+})
+export type KnowledgeArtifact = Schema.Schema.Type<typeof KnowledgeArtifactSchema>
+
+export type PutKnowledgeArtifactInput = {
+  readonly storageKey: string
+  readonly mediaType?: string
+  readonly bytes: Uint8Array
+}
+
+export type GetKnowledgeArtifactInput = {
+  readonly storageKey: string
+}
+
+export type KnowledgeArtifactStoreApi = {
+  readonly putArtifact: (input: PutKnowledgeArtifactInput) => Effect.Effect<void, KnowledgeArtifactError>
+  readonly getArtifact: (input: GetKnowledgeArtifactInput) => Effect.Effect<Uint8Array, KnowledgeArtifactError>
+  readonly deleteArtifact: (input: GetKnowledgeArtifactInput) => Effect.Effect<void, KnowledgeArtifactError>
+}
+
+export class KnowledgeArtifactStore extends Context.Service<KnowledgeArtifactStore, KnowledgeArtifactStoreApi>()(
+  '@yolk-sdk/knowledge/KnowledgeArtifactStore'
+) {}

package/src/chunking.ts

@@ -0,0 +1,175 @@
+import { Context, Effect, Layer } from 'effect'
+import { countTokens, decode, encode } from 'gpt-tokenizer/encoding/o200k_base'
+import type { KnowledgeChunk, KnowledgeMetadata } from './documents.ts'
+import { KnowledgeChunkingError } from './errors.ts'
+
+const SENTENCE_PATTERN = /[^.!?]+[.!?]+(?:["')\]]+)?\s*|[^.!?]+$/g
+const PARAGRAPH_BREAK_PATTERN = /(\n{2,})/
+const WORD_PATTERN = /\S+\s*/g
+
+export type ChunkKnowledgeDocumentInput = {
+  readonly collectionId: string
+  readonly documentId: string
+  readonly content: string
+  readonly maxTokens?: number
+  readonly metadata?: KnowledgeMetadata
+}
+
+export type KnowledgeChunkerApi = {
+  readonly chunk: (
+    input: ChunkKnowledgeDocumentInput
+  ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, KnowledgeChunkingError>
+}
+
+export class KnowledgeChunker extends Context.Service<KnowledgeChunker, KnowledgeChunkerApi>()(
+  '@yolk-sdk/knowledge/KnowledgeChunker'
+) {}
+
+export const countKnowledgeChunkTokens = (text: string) => countTokens(text)
+
+const sanitizeText = (text: string) =>
+  text
+    .replace(/\r\n/g, '\n')
+    .replace(/[\t ]+/g, ' ')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+
+const splitSentences = (text: string) =>
+  text
+    .split(PARAGRAPH_BREAK_PATTERN)
+    .flatMap(part => {
+      if (part.length === 0) {
+        return []
+      }
+
+      if (/^\n{2,}$/.test(part)) {
+        return ['\n\n']
+      }
+
+      return part.match(SENTENCE_PATTERN) ?? [part]
+    })
+    .filter(unit => unit.length > 0)
+
+const splitEncodedTokens = (text: string, maxTokens: number) => {
+  const tokens = encode(text)
+  return Array.from({ length: Math.ceil(tokens.length / maxTokens) }, (_, index) =>
+    decode(tokens.slice(index * maxTokens, index * maxTokens + maxTokens)).trim()
+  ).filter(chunk => chunk.length > 0)
+}
+
+const splitOversizedUnit = (text: string, maxTokens: number) => {
+  const words = text.match(WORD_PATTERN) ?? []
+  const chunks: string[] = []
+  let current = ''
+  let currentTokenCount = 0
+
+  for (const word of words) {
+    const wordTokenCount = countTokens(word)
+
+    if (wordTokenCount > maxTokens) {
+      const currentChunk = current.trim()
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk)
+      }
+
+      chunks.push(...splitEncodedTokens(word, maxTokens))
+      current = ''
+      currentTokenCount = 0
+      continue
+    }
+
+    if (currentTokenCount > 0 && currentTokenCount + wordTokenCount > maxTokens) {
+      const currentChunk = current.trim()
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk)
+      }
+
+      current = word
+      currentTokenCount = wordTokenCount
+      continue
+    }
+
+    current = `${current}${word}`
+    currentTokenCount += wordTokenCount
+  }
+
+  const finalChunk = current.trim()
+  if (finalChunk.length > 0) {
+    chunks.push(finalChunk)
+  }
+
+  return chunks
+}
+
+const splitUnitsBySize = (units: ReadonlyArray<string>, maxTokens: number) =>
+  units.flatMap(unit =>
+    countTokens(unit) <= maxTokens ? [unit] : splitOversizedUnit(unit, maxTokens)
+  )
+
+const buildChunkContents = (units: ReadonlyArray<string>, maxTokens: number) => {
+  const chunks: string[] = []
+  let current = ''
+  let currentTokenCount = 0
+
+  const pushCurrent = () => {
+    const content = current.trim()
+    if (content.length > 0) {
+      chunks.push(content)
+    }
+  }
+
+  for (const unit of units) {
+    const unitTokenCount = countTokens(unit)
+
+    if (currentTokenCount > 0 && currentTokenCount + unitTokenCount > maxTokens) {
+      pushCurrent()
+      current = unit
+      currentTokenCount = unitTokenCount
+      continue
+    }
+
+    current = `${current}${unit}`
+    currentTokenCount += unitTokenCount
+  }
+
+  pushCurrent()
+  return chunks
+}
+
+export const chunkKnowledgeText = (input: ChunkKnowledgeDocumentInput, maxTokens: number) =>
+  Effect.gen(function* () {
+    if (!Number.isInteger(maxTokens) || maxTokens < 1) {
+      return yield* Effect.fail(
+        new KnowledgeChunkingError({ message: 'Chunk maxTokens must be a positive integer' })
+      )
+    }
+
+    const sanitized = sanitizeText(input.content)
+    if (sanitized.length === 0) {
+      return yield* Effect.fail(new KnowledgeChunkingError({ message: 'Cannot chunk empty content' }))
+    }
+
+    const units = splitUnitsBySize(splitSentences(sanitized), maxTokens)
+    const contents = buildChunkContents(units, maxTokens)
+
+    if (contents.length === 0) {
+      return yield* Effect.fail(new KnowledgeChunkingError({ message: 'No chunks produced' }))
+    }
+
+    return contents.map((content, position) => ({
+      id: `${input.documentId}:chunk:${position}`,
+      collectionId: input.collectionId,
+      documentId: input.documentId,
+      content,
+      position,
+      tokenCount: countTokens(content),
+      metadata: input.metadata
+    })) satisfies ReadonlyArray<KnowledgeChunk>
+  })
+
+export const makeDefaultKnowledgeChunker = (input: { readonly maxTokens: number }): KnowledgeChunkerApi => ({
+  chunk: document => chunkKnowledgeText(document, document.maxTokens ?? input.maxTokens)
+})
+
+export const DefaultKnowledgeChunkerLive = (input: { readonly maxTokens: number } = { maxTokens: 512 }) =>
+  Layer.succeed(KnowledgeChunker, makeDefaultKnowledgeChunker(input))

package/src/context.ts

@@ -0,0 +1,42 @@
+import { Array as Arr } from 'effect'
+import type { KnowledgeRecord } from './records.ts'
+import type { KnowledgeRepresentation } from './representations.ts'
+
+export type KnowledgeContextItem = {
+  readonly record: KnowledgeRecord
+  readonly representation?: KnowledgeRepresentation
+}
+
+export type BuildKnowledgeContextInput = {
+  readonly items: ReadonlyArray<KnowledgeContextItem>
+  readonly maxCharacters: number
+}
+
+const truncationMarker = '\n\n[truncated: pinned knowledge exceeded context budget]'
+
+const recordHeader = (record: KnowledgeRecord) =>
+  `## ${record.title}\nrole: ${record.role}\nstatus: ${record.status}\ncontext: ${record.contextPolicy}`
+
+const itemBody = (item: KnowledgeContextItem) => {
+  const representationText = item.representation?.summary ?? item.representation?.contentText
+  return [recordHeader(item.record), item.record.summary, representationText]
+    .filter(section => section !== undefined && section.trim().length > 0)
+    .join('\n')
+}
+
+export const buildKnowledgeContext = (input: BuildKnowledgeContextInput) => {
+  const sections = Arr.map(input.items, itemBody).filter(section => section.trim().length > 0)
+  const header = '# Pinned knowledge\nUse this durable user knowledge as high-priority context. Cite knowledge records when relevant.'
+  const body = sections.join('\n\n')
+  const context = body.length === 0 ? '' : `${header}\n\n${body}`
+
+  if (context.length <= input.maxCharacters) {
+    return context
+  }
+
+  if (input.maxCharacters <= truncationMarker.length) {
+    return context.slice(0, input.maxCharacters).trimEnd()
+  }
+
+  return `${context.slice(0, input.maxCharacters - truncationMarker.length).trimEnd()}${truncationMarker}`
+}