@yolk-sdk/knowledge 0.1.0-canary.13 → 0.1.0-canary.130

package/src/documents.ts

@@ -1,109 +1,178 @@
-import { Schema } from 'effect'
+import * as Schema from 'effect/Schema'
 
-const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))
-const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))
-const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))
+export const NonEmptyTrimmedString = Schema.Trimmed.pipe(Schema.check(Schema.isNonEmpty()))
+
+export const NonNegativeInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))
+
+export const PositiveInteger = Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0)))
 
 export const KnowledgeMetadataSchema = Schema.Record(Schema.String, Schema.Unknown)
+
 export type KnowledgeMetadata = Schema.Schema.Type<typeof KnowledgeMetadataSchema>
 
-export const KnowledgeDocumentStatusSchema = Schema.Literals(['pending', 'processing', 'ready', 'error'])
+export const KnowledgeAvailabilitySchema = Schema.Literals(['pinned', 'searchable', 'archived'])
+
+export type KnowledgeAvailability = Schema.Schema.Type<typeof KnowledgeAvailabilitySchema>
+
+export const KnowledgeDocumentStatusSchema = Schema.Literals(['processing', 'ready', 'error'])
+
 export type KnowledgeDocumentStatus = Schema.Schema.Type<typeof KnowledgeDocumentStatusSchema>
 
-export const KnowledgeSourceSchema = Schema.Union([
-  Schema.Struct({
-    _tag: Schema.Literal('File'),
-    ref: NonEmptyTrimmedString,
-    name: Schema.optional(NonEmptyTrimmedString),
-    mediaType: Schema.optional(NonEmptyTrimmedString)
-  }),
-  Schema.Struct({
-    _tag: Schema.Literal('Url'),
-    url: NonEmptyTrimmedString
-  }),
-  Schema.Struct({
-    _tag: Schema.Literal('Text'),
-    label: Schema.optional(NonEmptyTrimmedString)
-  })
-])
-export type KnowledgeSource = Schema.Schema.Type<typeof KnowledgeSourceSchema>
+export const KnowledgeScopeId = NonEmptyTrimmedString.pipe(Schema.brand('KnowledgeScopeId'))
+
+export type KnowledgeScopeId = typeof KnowledgeScopeId.Type
 
-export const KnowledgeEmbeddingConfigSchema = Schema.Struct({
-  model: NonEmptyTrimmedString,
-  dimensions: PositiveInteger
+export const KnowledgeDocumentId = NonEmptyTrimmedString.pipe(Schema.brand('KnowledgeDocumentId'))
+
+export type KnowledgeDocumentId = typeof KnowledgeDocumentId.Type
+
+export const KnowledgeScopeSchema = Schema.Struct({
+  id: KnowledgeScopeId,
+  kind: Schema.optional(NonEmptyTrimmedString)
 })
-export type KnowledgeEmbeddingConfig = Schema.Schema.Type<typeof KnowledgeEmbeddingConfigSchema>
 
-export const KnowledgeChunkingConfigSchema = Schema.Struct({
-  strategy: Schema.Literal('sentence-token'),
-  maxTokens: PositiveInteger
+export type KnowledgeScope = Schema.Schema.Type<typeof KnowledgeScopeSchema>
+
+export const KnowledgeDocumentSchema = Schema.Struct({
+  id: KnowledgeDocumentId,
+  slug: NonEmptyTrimmedString,
+  title: NonEmptyTrimmedString,
+  purpose: NonEmptyTrimmedString,
+  origin: NonEmptyTrimmedString,
+  content: NonEmptyTrimmedString,
+  status: KnowledgeDocumentStatusSchema,
+  availability: KnowledgeAvailabilitySchema,
+  summary: Schema.optional(Schema.String),
+  errorMessage: Schema.optional(Schema.String),
+  reviewedAt: Schema.optional(Schema.DateTimeUtc),
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc,
+  updatedAt: Schema.DateTimeUtc
 })
-export type KnowledgeChunkingConfig = Schema.Schema.Type<typeof KnowledgeChunkingConfigSchema>
 
-export const KnowledgeCollectionSchema = Schema.Struct({
-  id: NonEmptyTrimmedString,
-  label: Schema.optional(NonEmptyTrimmedString),
-  embeddingConfig: KnowledgeEmbeddingConfigSchema,
-  chunkingConfig: KnowledgeChunkingConfigSchema,
+export type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>
+
+export const CreateKnowledgeDocumentInputSchema = Schema.Struct({
+  scope: KnowledgeScopeSchema,
+  slug: NonEmptyTrimmedString,
+  title: NonEmptyTrimmedString,
+  purpose: NonEmptyTrimmedString,
+  origin: NonEmptyTrimmedString,
+  content: NonEmptyTrimmedString,
+  availability: KnowledgeAvailabilitySchema,
+  summary: Schema.optional(Schema.String),
   metadata: Schema.optional(KnowledgeMetadataSchema)
 })
-export type KnowledgeCollection = Schema.Schema.Type<typeof KnowledgeCollectionSchema>
 
-export const KnowledgeDocumentSchema = Schema.Struct({
-  id: NonEmptyTrimmedString,
-  collectionId: NonEmptyTrimmedString,
-  source: KnowledgeSourceSchema,
-  status: KnowledgeDocumentStatusSchema,
+export type CreateKnowledgeDocumentInput = Schema.Schema.Type<
+  typeof CreateKnowledgeDocumentInputSchema
+>
+
+export const UpdateKnowledgeDocumentInputSchema = Schema.Struct({
+  scope: KnowledgeScopeSchema,
+  id: KnowledgeDocumentId,
+  slug: Schema.optional(NonEmptyTrimmedString),
   title: Schema.optional(NonEmptyTrimmedString),
+  purpose: Schema.optional(NonEmptyTrimmedString),
+  origin: Schema.optional(NonEmptyTrimmedString),
+  content: Schema.optional(NonEmptyTrimmedString),
+  status: Schema.optional(KnowledgeDocumentStatusSchema),
+  availability: Schema.optional(KnowledgeAvailabilitySchema),
   summary: Schema.optional(Schema.String),
   errorMessage: Schema.optional(Schema.String),
-  contentHash: Schema.optional(NonEmptyTrimmedString),
-  tokenCount: Schema.optional(NonNegativeInteger),
-  chunkCount: Schema.optional(NonNegativeInteger),
+  reviewedAt: Schema.optional(Schema.DateTimeUtc),
   metadata: Schema.optional(KnowledgeMetadataSchema)
 })
-export type KnowledgeDocument = Schema.Schema.Type<typeof KnowledgeDocumentSchema>
+
+export type UpdateKnowledgeDocumentInput = Schema.Schema.Type<
+  typeof UpdateKnowledgeDocumentInputSchema
+>
+
+export const KnowledgeFileSchema = Schema.Struct({
+  id: NonEmptyTrimmedString,
+  documentId: KnowledgeDocumentId,
+  storageKey: NonEmptyTrimmedString,
+  mediaType: Schema.optional(NonEmptyTrimmedString),
+  byteSize: Schema.optional(NonNegativeInteger),
+  checksum: Schema.optional(NonEmptyTrimmedString),
+  metadata: Schema.optional(KnowledgeMetadataSchema),
+  createdAt: Schema.DateTimeUtc
+})
+
+export type KnowledgeFile = Schema.Schema.Type<typeof KnowledgeFileSchema>
 
 export const KnowledgeChunkSchema = Schema.Struct({
   id: NonEmptyTrimmedString,
-  collectionId: NonEmptyTrimmedString,
-  documentId: NonEmptyTrimmedString,
+  scopeId: KnowledgeScopeId,
+  documentId: KnowledgeDocumentId,
   content: NonEmptyTrimmedString,
   position: NonNegativeInteger,
   tokenCount: PositiveInteger,
   metadata: Schema.optional(KnowledgeMetadataSchema)
 })
+
 export type KnowledgeChunk = Schema.Schema.Type<typeof KnowledgeChunkSchema>
 
+export const KnowledgeSearchScope = Schema.TaggedStruct('KnowledgeScope', {
+  id: KnowledgeScopeId
+})
+
+export const KnowledgeSearchScopes = Schema.TaggedStruct('KnowledgeScopes', {
+  ids: Schema.NonEmptyArray(KnowledgeScopeId)
+})
+
+export const KnowledgeSearchScopeSchema = Schema.Union([
+  KnowledgeSearchScope,
+  KnowledgeSearchScopes
+])
+
+export type KnowledgeSearchScope = typeof KnowledgeSearchScopeSchema.Type
+
+export const KnowledgeFileSource = Schema.TaggedStruct('File', {
+  ref: NonEmptyTrimmedString,
+  name: Schema.optional(NonEmptyTrimmedString),
+  mediaType: Schema.optional(NonEmptyTrimmedString)
+})
+
+export const KnowledgeUrlSource = Schema.TaggedStruct('Url', {
+  url: NonEmptyTrimmedString
+})
+
+export const KnowledgeTextSource = Schema.TaggedStruct('Text', {
+  label: Schema.optional(NonEmptyTrimmedString)
+})
+
+export const KnowledgeSourceSchema = Schema.Union([
+  KnowledgeFileSource,
+  KnowledgeUrlSource,
+  KnowledgeTextSource
+])
+
+export type KnowledgeSource = typeof KnowledgeSourceSchema.Type
+
+export const IndexedKnowledgeDocumentSchema = Schema.Struct({
+  id: KnowledgeDocumentId,
+  scopeId: KnowledgeScopeId,
+  source: KnowledgeSourceSchema,
+  status: KnowledgeDocumentStatusSchema,
+  title: Schema.optional(NonEmptyTrimmedString),
+  summary: Schema.optional(Schema.String),
+  errorMessage: Schema.optional(Schema.String),
+  contentHash: Schema.optional(NonEmptyTrimmedString),
+  tokenCount: Schema.optional(NonNegativeInteger),
+  chunkCount: Schema.optional(NonNegativeInteger),
+  metadata: Schema.optional(KnowledgeMetadataSchema)
+})
+
+export type IndexedKnowledgeDocument = Schema.Schema.Type<typeof IndexedKnowledgeDocumentSchema>
+
 export const ExtractedKnowledgeDocumentSchema = Schema.Struct({
   content: NonEmptyTrimmedString,
   title: Schema.optional(NonEmptyTrimmedString),
   summary: Schema.optional(Schema.String),
   metadata: Schema.optional(KnowledgeMetadataSchema)
 })
+
 export type ExtractedKnowledgeDocument = Schema.Schema.Type<typeof ExtractedKnowledgeDocumentSchema>
 
-export const KnowledgeSearchScopeSchema = Schema.Union([
-  Schema.Struct({ _tag: Schema.Literal('KnowledgeCollection'), id: NonEmptyTrimmedString }),
-  Schema.Struct({ _tag: Schema.Literal('KnowledgeCollections'), ids: Schema.Array(NonEmptyTrimmedString) })
-])
-export type KnowledgeSearchScope = Schema.Schema.Type<typeof KnowledgeSearchScopeSchema>
-
-export const defaultKnowledgeChunkingConfig: KnowledgeChunkingConfig = {
-  strategy: 'sentence-token',
-  maxTokens: 512
-}
-
-export const makeKnowledgeCollection = (input: {
-  readonly id: string
-  readonly label?: string
-  readonly embeddingConfig: KnowledgeEmbeddingConfig
-  readonly chunkingConfig?: KnowledgeChunkingConfig
-  readonly metadata?: KnowledgeMetadata
-}): KnowledgeCollection => ({
-  id: input.id,
-  label: input.label,
-  embeddingConfig: input.embeddingConfig,
-  chunkingConfig: input.chunkingConfig ?? defaultKnowledgeChunkingConfig,
-  metadata: input.metadata
-})
+export const defaultKnowledgeChunkMaxTokens = 512

package/src/errors.ts

@@ -38,7 +38,7 @@ export class KnowledgeSearchError extends Data.TaggedError('KnowledgeSearchError
   readonly cause?: unknown
 }> {}
 
-export class KnowledgeStoreError extends Schema.TaggedErrorClass<KnowledgeStoreError>()(
+export class KnowledgeStoreError extends Schema.TaggedError<KnowledgeStoreError>()(
   'KnowledgeStoreError',
   {
     message: Schema.String,
@@ -46,15 +46,15 @@ export class KnowledgeStoreError extends Schema.TaggedErrorClass<KnowledgeStoreE
   }
 ) {}
 
-export class KnowledgeArtifactError extends Schema.TaggedErrorClass<KnowledgeArtifactError>()(
-  'KnowledgeArtifactError',
+export class KnowledgeFileError extends Schema.TaggedError<KnowledgeFileError>()(
+  'KnowledgeFileError',
   {
     message: Schema.String,
     cause: Schema.optional(Schema.Unknown)
   }
 ) {}
 
-export class KnowledgeContextError extends Schema.TaggedErrorClass<KnowledgeContextError>()(
+export class KnowledgeContextError extends Schema.TaggedError<KnowledgeContextError>()(
   'KnowledgeContextError',
   {
     message: Schema.String,

package/src/extraction.ts

@@ -16,6 +16,7 @@ export type KnowledgeExtractorApi = {
   ) => Effect.Effect<ExtractedKnowledgeDocument, KnowledgeExtractionError>
 }
 
-export class KnowledgeExtractor extends Context.Service<KnowledgeExtractor, KnowledgeExtractorApi>()(
-  '@yolk-sdk/knowledge/KnowledgeExtractor'
-) {}
+export class KnowledgeExtractor extends Context.Service<
+  KnowledgeExtractor,
+  KnowledgeExtractorApi
+>()('@yolk-sdk/knowledge/KnowledgeExtractor') {}

package/src/files.ts

@@ -0,0 +1,24 @@
+import { Context } from 'effect'
+import type { Effect } from 'effect'
+import type { KnowledgeFileError } from './errors.ts'
+
+export type PutKnowledgeFileInput = {
+  readonly storageKey: string
+  readonly mediaType?: string
+  readonly bytes: Uint8Array
+}
+
+export type GetKnowledgeFileInput = {
+  readonly storageKey: string
+}
+
+export type KnowledgeFileBlobStoreApi = {
+  readonly putFile: (input: PutKnowledgeFileInput) => Effect.Effect<void, KnowledgeFileError>
+  readonly getFile: (input: GetKnowledgeFileInput) => Effect.Effect<Uint8Array, KnowledgeFileError>
+  readonly deleteFile: (input: GetKnowledgeFileInput) => Effect.Effect<void, KnowledgeFileError>
+}
+
+export class KnowledgeFileBlobStore extends Context.Service<
+  KnowledgeFileBlobStore,
+  KnowledgeFileBlobStoreApi
+>()('@yolk-sdk/knowledge/KnowledgeFileBlobStore') {}

package/src/index.ts

@@ -1,4 +1,7 @@
 export { buildKnowledgeContext } from './context.ts'
-export { KnowledgeStore } from './store.ts'
-export { KnowledgeArtifactStore } from './artifacts.ts'
-export { KnowledgeStoreError, KnowledgeArtifactError, KnowledgeContextError } from './errors.ts'
+
+export { KnowledgeStore, SearchIndexStore } from './store.ts'
+
+export { KnowledgeFileBlobStore } from './files.ts'
+
+export { KnowledgeStoreError, KnowledgeFileError, KnowledgeContextError } from './errors.ts'

package/src/ingestion.ts

@@ -1,24 +1,27 @@
 import { Array as Arr, Effect } from 'effect'
 import { KnowledgeChunker } from './chunking.ts'
+import { defaultKnowledgeChunkMaxTokens } from './documents.ts'
+import type { KnowledgeDocumentId, KnowledgeScopeId } from './documents.ts'
 import { KnowledgeEmbedder } from './embeddings.ts'
 import { KnowledgeExtractor } from './extraction.ts'
 import { KnowledgeSummarizer } from './summarization.ts'
 import type { LoadedKnowledgeSource } from './extraction.ts'
 import { KnowledgeIngestionError } from './errors.ts'
-import { SearchIndexStore } from './search-store.ts'
+import { SearchIndexStore } from './store.ts'
 
 export type IngestKnowledgeDocumentInput = {
-  readonly collectionId: string
-  readonly documentId: string
+  readonly scopeId: KnowledgeScopeId
+  readonly documentId: KnowledgeDocumentId
   readonly source: LoadedKnowledgeSource
   readonly contentHash?: string
+  readonly maxTokens?: number
 }
 
 const markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>
   Effect.gen(function* () {
     const store = yield* SearchIndexStore
     yield* store.markDocumentError({
-      collectionId: input.collectionId,
+      scopeId: input.scopeId,
       documentId: input.documentId,
       message: error.message
     })
@@ -27,7 +30,7 @@ const markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: Knowled
 export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
   Effect.gen(function* () {
     yield* Effect.annotateCurrentSpan({
-      'knowledge_search.set_id': input.collectionId,
+      'knowledge_search.scope_id': input.scopeId,
       'knowledge_search.document_id': input.documentId,
       'knowledge_search.source_type': input.source.source._tag
     })
@@ -36,19 +39,12 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
     const chunker = yield* KnowledgeChunker
     const embedder = yield* KnowledgeEmbedder
     const summarizer = yield* KnowledgeSummarizer
-    const collection = yield* store
-      .getSet(input.collectionId)
-      .pipe(
-        Effect.mapError(
-          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
-        )
-      )
 
     yield* store
       .upsertDocument({
         document: {
           id: input.documentId,
-          collectionId: input.collectionId,
+          scopeId: input.scopeId,
           source: input.source.source,
           status: 'processing',
           metadata: input.source.metadata
@@ -56,7 +52,8 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
       })
       .pipe(
         Effect.mapError(
-          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+          error =>
+            new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
         )
       )
 
@@ -64,21 +61,23 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
       .extract(input.source)
       .pipe(
         Effect.mapError(
-          error => new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })
+          error =>
+            new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })
         )
       )
 
     const chunks = yield* chunker
       .chunk({
-        collectionId: input.collectionId,
+        scopeId: input.scopeId,
         documentId: input.documentId,
         content: extracted.content,
-        maxTokens: collection.chunkingConfig.maxTokens,
+        maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,
         metadata: extracted.metadata
       })
       .pipe(
         Effect.mapError(
-          error => new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })
+          error =>
+            new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })
         )
       )
 
@@ -86,7 +85,12 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
       {
         embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(
           Effect.mapError(
-            error => new KnowledgeIngestionError({ message: error.message, stage: 'embed', cause: error })
+            error =>
+              new KnowledgeIngestionError({
+                message: error.message,
+                stage: 'embed',
+                cause: error
+              })
           )
         ),
         summary: summarizer
@@ -98,7 +102,11 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
           .pipe(
             Effect.mapError(
               error =>
-                new KnowledgeIngestionError({ message: error.message, stage: 'summarize', cause: error })
+                new KnowledgeIngestionError({
+                  message: error.message,
+                  stage: 'summarize',
+                  cause: error
+                })
             )
           )
       },
@@ -107,7 +115,10 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
 
     if (indexed.embeddings.length !== chunks.length) {
       return yield* Effect.fail(
-        new KnowledgeIngestionError({ message: 'Embedding count did not match chunk count', stage: 'embed' })
+        new KnowledgeIngestionError({
+          message: 'Embedding count did not match chunk count',
+          stage: 'embed'
+        })
       )
     }
 
@@ -115,22 +126,25 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
       chunk,
       embedding
     }))
+
     yield* store
       .replaceDocumentChunks({
-        collectionId: input.collectionId,
+        scopeId: input.scopeId,
         documentId: input.documentId,
         chunks: indexedChunks
       })
       .pipe(
         Effect.mapError(
-          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+          error =>
+            new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
         )
       )
 
     const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)
+
     return yield* store
       .markDocumentReady({
-        collectionId: input.collectionId,
+        scopeId: input.scopeId,
         documentId: input.documentId,
         title: indexed.summary.title ?? extracted.title,
         summary: indexed.summary.summary ?? extracted.summary,
@@ -140,16 +154,21 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
       })
       .pipe(
         Effect.mapError(
-          error => new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
+          error =>
+            new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })
         )
       )
   }).pipe(
     Effect.withSpan('knowledge_search.ingestDocument'),
-    Effect.catch(error => markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error))))
+    Effect.catch(error =>
+      markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))
+    )
   )
 
 export type KnowledgeIngestionPipeline = {
-  readonly ingest: (input: IngestKnowledgeDocumentInput) => ReturnType<typeof ingestKnowledgeDocument>
+  readonly ingest: (
+    input: IngestKnowledgeDocumentInput
+  ) => ReturnType<typeof ingestKnowledgeDocument>
 }
 
 export const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({