@yolk-sdk/knowledge 0.0.1-canary.0

package/src/documents.ts

@@ -0,0 +1,109 @@
+import { Schema } from 'effect'
+
+const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))
+const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))
+const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))
+
+export const KnowledgeMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)
+export type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>
+
+export const KnowledgeDocumentStatusSchema = Schema.Literals(['pending', 'processing', 'ready', 'error'])
+export type KnowledgeDocumentStatus = Schema.Schema.Type<typeof KnowledgeDocumentStatusSchema>
+
+export const KnowledgeSourceSchema = Schema.Union([
+  Schema.Struct({
+    _tag: Schema.Literal('File'),
+    ref: NonEmptyTrimmedString,
+    name: Schema.optional(NonEmptyTrimmedString),
+    mediaType: Schema.optional(NonEmptyTrimmedString)
+  }),
+  Schema.Struct({
+    _tag: Schema.Literal('Url'),
+    url: NonEmptyTrimmedString
+  }),
+  Schema.Struct({
+    _tag: Schema.Literal('Text'),
+    label: Schema.optional(NonEmptyTrimmedString)
+  })
+])
+export type KnowledgeSource = Schema.Schema.Type<typeof KnowledgeSourceSchema>
+
+export const KnowledgeEmbeddingConfigSchema = Schema.Struct({
+  model: NonEmptyTrimmedString,
+  dimensions: PositiveInteger
+})
+export type KnowledgeEmbeddingConfig = Schema.Schema.Type<typeof KnowledgeEmbeddingConfigSchema>
+
+export const KnowledgeChunkingConfigSchema = Schema.Struct({
+  strategy: Schema.Literal('sentence-token'),
+  maxTokens: PositiveInteger
+})
+export type KnowledgeChunkingConfig = Schema.Schema.Type<typeof KnowledgeChunkingConfigSchema>
+
+export const KnowledgeCollectionSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  label: Schema.optional(NonEmptyTrimmedString),
+  embeddingConfig: KnowledgeEmbeddingConfigSchema,
+  chunkingConfig: KnowledgeChunkingConfigSchema,
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+export type KnowledgeCollection = Schema.Schema.Type<typeof KnowledgeCollectionSchema>
+
+export const KnowledgeDocumentSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  collectionId: NonEmptyTrimmedString,
+  source: KnowledgeSourceSchema,
+  status: KnowledgeDocumentStatusSchema,
+  title: Schema.optional(NonEmptyTrimmedString),
+  summary: Schema.optional(Schema.String),
+  errorMessage: Schema.optional(Schema.String),
+  contentHash: Schema.optional(NonEmptyTrimmedString),
+  tokenCount: Schema.optional(NonNegativeInteger),
+  chunkCount: Schema.optional(NonNegativeInteger),
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+export type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>
+
+export const KnowledgeChunkSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  collectionId: NonEmptyTrimmedString,
+  documentId: NonEmptyTrimmedString,
+  content: NonEmptyTrimmedString,
+  position: NonNegativeInteger,
+  tokenCount: PositiveInteger,
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+export type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>
+
+export const ExtractedKnowledgeDocumentSchema = Schema.Struct({
+  content: NonEmptyTrimmedString,
+  title: Schema.optional(NonEmptyTrimmedString),
+  summary: Schema.optional(Schema.String),
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+export type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>
+
+export const KnowledgeSearchScopeSchema = Schema.Union([
+  Schema.Struct({ _tag: Schema.Literal('KnowledgeCollection'), id: NonEmptyTrimmedString }),
+  Schema.Struct({ _tag: Schema.Literal('KnowledgeCollections'), ids: Schema.Array(NonEmptyTrimmedString) })
+])
+export type KnowledgeSearchScope = Schema.Schema.Type<typeof KnowledgeSearchScopeSchema>
+
+export const defaultKnowledgeChunkingConfig: KnowledgeChunkingConfig = {
+  strategy: 'sentence-token',
+  maxTokens: 512
+}
+
+export const makeKnowledgeCollection = (input: {
+  readonly id: string
+  readonly label?: string
+  readonly embeddingConfig: KnowledgeEmbeddingConfig
+  readonly chunkingConfig?: KnowledgeChunkingConfig
+  readonly metadata?: KnowledgeMetadata
+}): KnowledgeCollection => ({
+  id: input.id,
+  label: input.label,
+  embeddingConfig: input.embeddingConfig,
+  chunkingConfig: input.chunkingConfig ?? defaultKnowledgeChunkingConfig,
+  metadata: input.metadata
+})

package/src/embeddings.ts

@@ -0,0 +1,18 @@
+import { Context } from 'effect'
+import type { Effect } from 'effect'
+import type { KnowledgeEmbeddingError } from './errors.ts'
+
+export type KnowledgeEmbedding = ReadonlyArray<number>
+
+export type KnowledgeEmbedderApi = {
+  readonly embedTexts: (
+    texts: ReadonlyArray<string>
+  ) => Effect.Effect<ReadonlyArray<KnowledgeEmbedding>, KnowledgeEmbeddingError>
+  readonly embedQuery: (query: string) => Effect.Effect<KnowledgeEmbedding, KnowledgeEmbeddingError>
+}
+
+export class KnowledgeEmbedder extends Context.Service<KnowledgeEmbedder, KnowledgeEmbedderApi>()(
+  '@yolk-sdk/knowledge/KnowledgeEmbedder'
+) {}
+
+export { KnowledgeEmbeddingError as EmbedderError } from './errors.ts'

package/src/errors.ts

@@ -0,0 +1,63 @@
+import { Data } from 'effect'
+import * as Schema from 'effect/Schema'
+
+export class SearchIndexStoreError extends Data.TaggedError('SearchIndexStoreError')<{
+  readonly message: string
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeExtractionError extends Data.TaggedError('KnowledgeExtractionError')<{
+  readonly message: string
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeChunkingError extends Data.TaggedError('KnowledgeChunkingError')<{
+  readonly message: string
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeEmbeddingError extends Data.TaggedError('KnowledgeEmbeddingError')<{
+  readonly message: string
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeSummarizationError extends Data.TaggedError('KnowledgeSummarizationError')<{
+  readonly message: string
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeIngestionError extends Data.TaggedError('KnowledgeIngestionError')<{
+  readonly message: string
+  readonly stage: 'store' | 'extract' | 'chunk' | 'embed' | 'summarize'
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeSearchError extends Data.TaggedError('KnowledgeSearchError')<{
+  readonly message: string
+  readonly stage: 'store' | 'embed'
+  readonly cause?: unknown
+}> {}
+
+export class KnowledgeStoreError extends Schema.TaggedErrorClass<KnowledgeStoreError>()(
+  'KnowledgeStoreError',
+  {
+    message: Schema.String,
+    cause: Schema.optional(Schema.Unknown)
+  }
+) {}
+
+export class KnowledgeArtifactError extends Schema.TaggedErrorClass<KnowledgeArtifactError>()(
+  'KnowledgeArtifactError',
+  {
+    message: Schema.String,
+    cause: Schema.optional(Schema.Unknown)
+  }
+) {}
+
+export class KnowledgeContextError extends Schema.TaggedErrorClass<KnowledgeContextError>()(
+  'KnowledgeContextError',
+  {
+    message: Schema.String,
+    cause: Schema.optional(Schema.Unknown)
+  }
+) {}

package/src/extraction.ts

@@ -0,0 +1,21 @@
+import { Context } from 'effect'
+import type { Effect } from 'effect'
+import type { ExtractedKnowledgeDocument, KnowledgeMetadata, KnowledgeSource } from './documents.ts'
+import type { KnowledgeExtractionError } from './errors.ts'
+
+export type LoadedKnowledgeSource = {
+  readonly source: KnowledgeSource
+  readonly content: string | Uint8Array
+  readonly mediaType?: string
+  readonly metadata?: KnowledgeMetadata
+}
+
+export type KnowledgeExtractorApi = {
+  readonly extract: (
+    source: LoadedKnowledgeSource
+  ) => Effect.Effect<ExtractedKnowledgeDocument, KnowledgeExtractionError>
+}
+
+export class KnowledgeExtractor extends Context.Service<KnowledgeExtractor, KnowledgeExtractorApi>()(
+  '@yolk-sdk/knowledge/KnowledgeExtractor'
+) {}

package/src/index.ts

@@ -0,0 +1,4 @@
+export { buildKnowledgeContext } from './context.ts'
+export { KnowledgeStore } from './store.ts'
+export { KnowledgeArtifactStore } from './artifacts.ts'
+export { KnowledgeStoreError, KnowledgeArtifactError, KnowledgeContextError } from './errors.ts'

package/src/ingestion.ts

@@ -0,0 +1,157 @@
+import { Array as Arr, Effect } from 'effect'
+import { KnowledgeChunker } from './chunking.ts'
+import { KnowledgeEmbedder } from './embeddings.ts'
+import { KnowledgeExtractor } from './extraction.ts'
+import { KnowledgeSummarizer } from './summarization.ts'
+import type { LoadedKnowledgeSource } from './extraction.ts'
+import { KnowledgeIngestionError } from './errors.ts'
+import { SearchIndexStore } from './search-store.ts'
+
+export type IngestKnowledgeDocumentInput = {
+  readonly collectionId: string
+  readonly documentId: string
+  readonly source: LoadedKnowledgeSource
+  readonly contentHash?: string
+}
+
+const markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>
+  Effect.gen(function* () {
+    const store = yield* SearchIndexStore
+    yield* store.markDocumentError({
+      collectionId: input.collectionId,
+      documentId: input.documentId,
+      message: error.message
+    })
+  }).pipe(Effect.catch(() => Effect.void))
+
+export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
+  Effect.gen(function* () {
+    yield* Effect.annotateCurrentSpan({
+      'knowledge_search.set_id': input.collectionId,
+      'knowledge_search.document_id': input.documentId,
+      'knowledge_search.source_type': input.source.source._tag
+    })
+    const store = yield* SearchIndexStore
+    const extractor = yield* KnowledgeExtractor
+    const chunker = yield* KnowledgeChunker
+    const embedder = yield* KnowledgeEmbedder
+    const summarizer = yield* KnowledgeSummarizer
+    const collection = yield* store
+      .getSet(input.collectionId)
+      .pipe(
+        Effect.mapError(
+          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+        )
+      )
+
+    yield* store
+      .upsertDocument({
+        document: {
+          id: input.documentId,
+          collectionId: input.collectionId,
+          source: input.source.source,
+          status: 'processing',
+          metadata: input.source.metadata
+        }
+      })
+      .pipe(
+        Effect.mapError(
+          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+        )
+      )
+
+    const extracted = yield* extractor
+      .extract(input.source)
+      .pipe(
+        Effect.mapError(
+          error => new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })
+        )
+      )
+
+    const chunks = yield* chunker
+      .chunk({
+        collectionId: input.collectionId,
+        documentId: input.documentId,
+        content: extracted.content,
+        maxTokens: collection.chunkingConfig.maxTokens,
+        metadata: extracted.metadata
+      })
+      .pipe(
+        Effect.mapError(
+          error => new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })
+        )
+      )
+
+    const indexed = yield* Effect.all(
+      {
+        embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(
+          Effect.mapError(
+            error => new KnowledgeIngestionError({ message: error.message, stage: 'embed', cause: error })
+          )
+        ),
+        summary: summarizer
+          .summarize({
+            content: extracted.content,
+            sourceTitle: extracted.title,
+            metadata: extracted.metadata
+          })
+          .pipe(
+            Effect.mapError(
+              error =>
+                new KnowledgeIngestionError({ message: error.message, stage: 'summarize', cause: error })
+            )
+          )
+      },
+      { concurrency: 'unbounded' }
+    )
+
+    if (indexed.embeddings.length !== chunks.length) {
+      return yield* Effect.fail(
+        new KnowledgeIngestionError({ message: 'Embedding count did not match chunk count', stage: 'embed' })
+      )
+    }
+
+    const indexedChunks = Arr.zip(chunks, indexed.embeddings).map(([chunk, embedding]) => ({
+      chunk,
+      embedding
+    }))
+    yield* store
+      .replaceDocumentChunks({
+        collectionId: input.collectionId,
+        documentId: input.documentId,
+        chunks: indexedChunks
+      })
+      .pipe(
+        Effect.mapError(
+          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+        )
+      )
+
+    const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)
+    return yield* store
+      .markDocumentReady({
+        collectionId: input.collectionId,
+        documentId: input.documentId,
+        title: indexed.summary.title ?? extracted.title,
+        summary: indexed.summary.summary ?? extracted.summary,
+        contentHash: input.contentHash,
+        tokenCount,
+        chunkCount: chunks.length
+      })
+      .pipe(
+        Effect.mapError(
+          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+        )
+      )
+  }).pipe(
+    Effect.withSpan('knowledge_search.ingestDocument'),
+    Effect.catch(error => markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error))))
+  )
+
+export type KnowledgeIngestionPipeline = {
+  readonly ingest: (input: IngestKnowledgeDocumentInput) => ReturnType<typeof ingestKnowledgeDocument>
+}
+
+export const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({
+  ingest: ingestKnowledgeDocument
+})

package/src/links.ts

@@ -0,0 +1,23 @@
+import * as Schema from 'effect/Schema'
+import { KnowledgeMetadataSchema, NonEmptyTrimmedString } from './records.ts'
+
+export const KnowledgeLinkTypeSchema = Schema.Literals([
+  'cites',
+  'supports',
+  'contradicts',
+  'supersedes',
+  'mentions',
+  'derived_from',
+  'related_to'
+])
+export type KnowledgeLinkType = Schema.Schema.Type<typeof KnowledgeLinkTypeSchema>
+
+export const KnowledgeLinkSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  fromRecordId: NonEmptyTrimmedString,
+  toRecordId: NonEmptyTrimmedString,
+  type: KnowledgeLinkTypeSchema,
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc
+})
+export type KnowledgeLink = Schema.Schema.Type<typeof KnowledgeLinkSchema>

package/src/provenance.ts

@@ -0,0 +1,25 @@
+import * as Schema from 'effect/Schema'
+import { KnowledgeMetadataSchema, NonEmptyTrimmedString } from './records.ts'
+
+export const KnowledgeProvenanceSourceKindSchema = Schema.Literals([
+  'upload',
+  'user_statement',
+  'url',
+  'generated',
+  'imported',
+  'external_api'
+])
+export type KnowledgeProvenanceSourceKind = Schema.Schema.Type<typeof KnowledgeProvenanceSourceKindSchema>
+
+export const KnowledgeProvenanceSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  recordId: NonEmptyTrimmedString,
+  artifactId: Schema.optional(NonEmptyTrimmedString),
+  sourceKind: KnowledgeProvenanceSourceKindSchema,
+  sourceLabel: NonEmptyTrimmedString,
+  sourceUrl: Schema.optional(NonEmptyTrimmedString),
+  observedAt: Schema.optional(Schema.DateTimeUtc),
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc
+})
+export type KnowledgeProvenance = Schema.Schema.Type<typeof KnowledgeProvenanceSchema>

package/src/records.ts

@@ -0,0 +1,76 @@
+import * as Schema from 'effect/Schema'
+
+export const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))
+export const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))
+export const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))
+
+export const KnowledgeMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)
+export type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>
+
+export const KnowledgeRecordRoleSchema = Schema.Literals([
+  'source',
+  'note',
+  'operating_protocol',
+  'knowledge_map',
+  'compiled_truth',
+  'decision'
+])
+export type KnowledgeRecordRole = Schema.Schema.Type<typeof KnowledgeRecordRoleSchema>
+
+export const KnowledgeContextPolicySchema = Schema.Literals([
+  'pinned',
+  'routable',
+  'searchable',
+  'archival'
+])
+export type KnowledgeContextPolicy = Schema.Schema.Type<typeof KnowledgeContextPolicySchema>
+
+export const KnowledgeLifecycleStatusSchema = Schema.Literals([
+  'draft',
+  'processing',
+  'ready',
+  'error',
+  'archived',
+  'deleted'
+])
+export type KnowledgeLifecycleStatus = Schema.Schema.Type<typeof KnowledgeLifecycleStatusSchema>
+
+export const KnowledgeScopeSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  kind: Schema.optional(NonEmptyTrimmedString)
+})
+export type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>
+
+export const KnowledgeRecordSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  role: KnowledgeRecordRoleSchema,
+  title: NonEmptyTrimmedString,
+  status: KnowledgeLifecycleStatusSchema,
+  contextPolicy: KnowledgeContextPolicySchema,
+  summary: Schema.optional(Schema.String),
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc,
+  updatedAt: Schema.DateTimeUtc
+})
+export type KnowledgeRecord = Schema.Schema.Type<typeof KnowledgeRecordSchema>
+
+export const CreateKnowledgeRecordInputSchema = Schema.Struct({
+  scope: KnowledgeScopeSchema,
+  role: KnowledgeRecordRoleSchema,
+  title: NonEmptyTrimmedString,
+  contextPolicy: KnowledgeContextPolicySchema,
+  summary: Schema.optional(Schema.String),
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+export type CreateKnowledgeRecordInput = Schema.Schema.Type<typeof CreateKnowledgeRecordInputSchema>
+
+export const UpdateKnowledgeRecordInputSchema = Schema.Struct({
+  scope: KnowledgeScopeSchema,
+  id: NonEmptyTrimmedString,
+  title: Schema.optional(NonEmptyTrimmedString),
+  status: Schema.optional(KnowledgeLifecycleStatusSchema),
+  contextPolicy: Schema.optional(KnowledgeContextPolicySchema),
+  summary: Schema.optional(Schema.String),
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+export type UpdateKnowledgeRecordInput = Schema.Schema.Type<typeof UpdateKnowledgeRecordInputSchema>

package/src/representations.ts

@@ -0,0 +1,51 @@
+import * as Schema from 'effect/Schema'
+import {
+  KnowledgeMetadataSchema,
+  NonEmptyTrimmedString,
+  NonNegativeInteger
+} from './records.ts'
+
+export const KnowledgeRepresentationModalitySchema = Schema.Literals([
+  'text',
+  'image',
+  'audio',
+  'video',
+  'table'
+])
+export type KnowledgeRepresentationModality = Schema.Schema.Type<typeof KnowledgeRepresentationModalitySchema>
+
+export const KnowledgeRepresentationStatusSchema = Schema.Literals([
+  'pending',
+  'processing',
+  'ready',
+  'error'
+])
+export type KnowledgeRepresentationStatus = Schema.Schema.Type<typeof KnowledgeRepresentationStatusSchema>
+
+export const KnowledgeRepresentationSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  recordId: NonEmptyTrimmedString,
+  artifactId: Schema.optional(NonEmptyTrimmedString),
+  modality: KnowledgeRepresentationModalitySchema,
+  status: KnowledgeRepresentationStatusSchema,
+  contentText: Schema.optional(Schema.String),
+  summary: Schema.optional(Schema.String),
+  model: Schema.optional(NonEmptyTrimmedString),
+  errorMessage: Schema.optional(Schema.String),
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc,
+  updatedAt: Schema.DateTimeUtc
+})
+export type KnowledgeRepresentation = Schema.Schema.Type<typeof KnowledgeRepresentationSchema>
+
+export const KnowledgeRepresentationChunkSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  recordId: NonEmptyTrimmedString,
+  representationId: NonEmptyTrimmedString,
+  content: NonEmptyTrimmedString,
+  position: NonNegativeInteger,
+  tokenCount: NonNegativeInteger,
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc
+})
+export type KnowledgeRepresentationChunk = Schema.Schema.Type<typeof KnowledgeRepresentationChunkSchema>

package/src/search-store.ts

@@ -0,0 +1,98 @@
+import { Context } from 'effect'
+import type { Effect } from 'effect'
+import type { KnowledgeChunk, KnowledgeDocument, KnowledgeSearchScope, KnowledgeCollection } from './documents.ts'
+import type { KnowledgeEmbedding } from './embeddings.ts'
+import type { SearchIndexStoreError } from './errors.ts'
+
+export type UpsertKnowledgeDocumentInput = {
+  readonly document: KnowledgeDocument
+}
+
+export type ReplaceKnowledgeDocumentChunksInput = {
+  readonly collectionId: string
+  readonly documentId: string
+  readonly chunks: ReadonlyArray<{
+    readonly chunk: KnowledgeChunk
+    readonly embedding: KnowledgeEmbedding
+  }>
+}
+
+export type MarkKnowledgeDocumentReadyInput = {
+  readonly collectionId: string
+  readonly documentId: string
+  readonly title?: string
+  readonly summary?: string
+  readonly contentHash?: string
+  readonly tokenCount: number
+  readonly chunkCount: number
+}
+
+export type MarkKnowledgeDocumentErrorInput = {
+  readonly collectionId: string
+  readonly documentId: string
+  readonly message: string
+}
+
+export type KnowledgeChunkSearchInput = {
+  readonly scope: KnowledgeSearchScope
+  readonly embedding: KnowledgeEmbedding
+  readonly limit: number
+  readonly minScore?: number
+}
+
+export type KnowledgeChunkTextSearchInput = {
+  readonly scope: KnowledgeSearchScope
+  readonly query: string
+  readonly limit: number
+}
+
+export type KnowledgeChunkSearchResult = {
+  readonly chunk: KnowledgeChunk
+  readonly score: number
+  readonly document: KnowledgeDocument
+}
+
+export type KnowledgeSearchContextChunksInput = {
+  readonly collectionId: string
+  readonly documentId: string
+  readonly position: number
+  readonly contextChunks: number
+}
+
+export type SearchIndexStoreApi = {
+  readonly upsertSet: (set: KnowledgeCollection) => Effect.Effect<KnowledgeCollection, SearchIndexStoreError>
+  readonly getSet: (id: string) => Effect.Effect<KnowledgeCollection, SearchIndexStoreError>
+  readonly upsertDocument: (
+    input: UpsertKnowledgeDocumentInput
+  ) => Effect.Effect<KnowledgeDocument, SearchIndexStoreError>
+  readonly markDocumentProcessing: (input: {
+    readonly collectionId: string
+    readonly documentId: string
+  }) => Effect.Effect<KnowledgeDocument, SearchIndexStoreError>
+  readonly replaceDocumentChunks: (
+    input: ReplaceKnowledgeDocumentChunksInput
+  ) => Effect.Effect<void, SearchIndexStoreError>
+  readonly markDocumentReady: (
+    input: MarkKnowledgeDocumentReadyInput
+  ) => Effect.Effect<KnowledgeDocument, SearchIndexStoreError>
+  readonly markDocumentError: (
+    input: MarkKnowledgeDocumentErrorInput
+  ) => Effect.Effect<void, SearchIndexStoreError>
+  readonly deleteDocument: (input: {
+    readonly collectionId: string
+    readonly documentId: string
+  }) => Effect.Effect<void, SearchIndexStoreError>
+  readonly searchChunks: (
+    input: KnowledgeChunkSearchInput
+  ) => Effect.Effect<ReadonlyArray<KnowledgeChunkSearchResult>, SearchIndexStoreError>
+  readonly searchChunksByText: (
+    input: KnowledgeChunkTextSearchInput
+  ) => Effect.Effect<ReadonlyArray<KnowledgeChunkSearchResult>, SearchIndexStoreError>
+  readonly getContextChunks: (
+    input: KnowledgeSearchContextChunksInput
+  ) => Effect.Effect<ReadonlyArray<KnowledgeChunk>, SearchIndexStoreError>
+}
+
+export class SearchIndexStore extends Context.Service<SearchIndexStore, SearchIndexStoreApi>()(
+  '@yolk-sdk/knowledge/SearchIndexStore'
+) {}