@yolk-sdk/knowledge 0.1.0-canary.36 → 0.1.0-canary.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ingestion.d.mts.map +1 -1
- package/dist/ingestion.mjs.map +1 -1
- package/package.json +2 -2
- package/src/ingestion.ts +10 -12
package/dist/ingestion.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion.d.mts","names":[],"sources":["../src/ingestion.ts"],"mappings":";;;;;;;;;KAUY,4BAAA;EAAA,SACD,OAAA;EAAA,SACA,UAAA;EAAA,SACA,MAAA,EAAQ,qBAAqB;EAAA,SAC7B,WAAA;EAAA,SACA,SAAA;AAAA;AAAA,cAaE,uBAAA,GAA2B,KAAA,EAAO,4BAAA,KAA4B,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"ingestion.d.mts","names":[],"sources":["../src/ingestion.ts"],"mappings":";;;;;;;;;KAUY,4BAAA;EAAA,SACD,OAAA;EAAA,SACA,UAAA;EAAA,SACA,MAAA,EAAQ,qBAAqB;EAAA,SAC7B,WAAA;EAAA,SACA,SAAA;AAAA;AAAA,cAaE,uBAAA,GAA2B,KAAA,EAAO,4BAAA,KAA4B,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;KAwI/D,0BAAA;EAAA,SACD,MAAA,GACP,KAAA,EAAO,4BAAA,KACJ,UAAA,QAAkB,uBAAA;AAAA;AAAA,cAGZ,qBAAA,QAA4B,0BAEvC"}
|
package/dist/ingestion.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { defaultKnowledgeChunkMaxTokens } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type IngestKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly source: LoadedKnowledgeSource\n readonly contentHash?: string\n readonly maxTokens?: number\n}\n\nconst markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n yield* store.markDocumentError({\n scopeId: input.scopeId,\n documentId: input.documentId,\n message: error.message\n })\n }).pipe(Effect.catch(() => Effect.void))\n\nexport const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>\n Effect.gen(function* () {\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.scope_id': input.scopeId,\n 'knowledge_search.document_id': input.documentId,\n 'knowledge_search.source_type': input.source.source._tag\n })\n const store = yield* SearchIndexStore\n const extractor = yield* KnowledgeExtractor\n const chunker = yield* KnowledgeChunker\n const embedder = yield* KnowledgeEmbedder\n const summarizer = yield* KnowledgeSummarizer\n\n yield* store\n .upsertDocument({\n document: {\n id: input.documentId,\n scopeId: input.scopeId,\n source: input.source.source,\n status: 'processing',\n metadata: input.source.metadata\n }\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const extracted = yield* extractor\n .extract(input.source)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })\n )\n )\n\n const chunks = yield* chunker\n .chunk({\n scopeId: input.scopeId,\n documentId: input.documentId,\n content: extracted.content,\n maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })\n )\n )\n\n const indexed = yield* Effect.all(\n {\n embeddings: embedder
|
|
1
|
+
{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { defaultKnowledgeChunkMaxTokens } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type IngestKnowledgeDocumentInput = {\n readonly scopeId: string\n readonly documentId: string\n readonly source: LoadedKnowledgeSource\n readonly contentHash?: string\n readonly maxTokens?: number\n}\n\nconst markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n yield* store.markDocumentError({\n scopeId: input.scopeId,\n documentId: input.documentId,\n message: error.message\n })\n }).pipe(Effect.catch(() => Effect.void))\n\nexport const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>\n Effect.gen(function* () {\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.scope_id': input.scopeId,\n 'knowledge_search.document_id': input.documentId,\n 'knowledge_search.source_type': input.source.source._tag\n })\n const store = yield* SearchIndexStore\n const extractor = yield* KnowledgeExtractor\n const chunker = yield* KnowledgeChunker\n const embedder = yield* KnowledgeEmbedder\n const summarizer = yield* KnowledgeSummarizer\n\n yield* store\n .upsertDocument({\n document: {\n id: input.documentId,\n scopeId: input.scopeId,\n source: input.source.source,\n status: 'processing',\n metadata: input.source.metadata\n }\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const extracted = yield* extractor\n .extract(input.source)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })\n )\n )\n\n const chunks = yield* chunker\n .chunk({\n scopeId: input.scopeId,\n documentId: input.documentId,\n content: extracted.content,\n maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })\n )\n )\n\n const indexed = yield* Effect.all(\n {\n embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'embed',\n cause: error\n })\n )\n ),\n summary: summarizer\n .summarize({\n content: extracted.content,\n sourceTitle: extracted.title,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'summarize',\n cause: error\n })\n )\n )\n },\n { concurrency: 'unbounded' }\n )\n\n if (indexed.embeddings.length !== chunks.length) {\n return yield* Effect.fail(\n new KnowledgeIngestionError({\n message: 'Embedding count did not match chunk count',\n stage: 'embed'\n })\n )\n }\n\n const indexedChunks = Arr.zip(chunks, indexed.embeddings).map(([chunk, embedding]) => ({\n chunk,\n embedding\n }))\n yield* store\n .replaceDocumentChunks({\n scopeId: input.scopeId,\n documentId: input.documentId,\n chunks: indexedChunks\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)\n return yield* store\n .markDocumentReady({\n scopeId: input.scopeId,\n documentId: input.documentId,\n title: indexed.summary.title ?? extracted.title,\n summary: indexed.summary.summary ?? extracted.summary,\n contentHash: input.contentHash,\n tokenCount,\n chunkCount: chunks.length\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n }).pipe(\n Effect.withSpan('knowledge_search.ingestDocument'),\n Effect.catch(error =>\n markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))\n )\n )\n\nexport type KnowledgeIngestionPipeline = {\n readonly ingest: (\n input: IngestKnowledgeDocumentInput\n ) => ReturnType<typeof ingestKnowledgeDocument>\n}\n\nexport const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({\n ingest: ingestKnowledgeDocument\n})\n"],"mappings":";;;;;;;;;AAkBA,MAAM,uBAAuB,OAAqC,UAChE,OAAO,IAAI,aAAa;CAEtB,QAAO,OADc,kBACR,kBAAkB;EAC7B,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,MAAM;CACjB,CAAC;AACH,CAAC,EAAE,KAAK,OAAO,YAAY,OAAO,IAAI,CAAC;AAEzC,MAAa,2BAA2B,UACtC,OAAO,IAAI,aAAa;CACtB,OAAO,OAAO,oBAAoB;EAChC,6BAA6B,MAAM;EACnC,gCAAgC,MAAM;EACtC,gCAAgC,MAAM,OAAO,OAAO;CACtD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,YAAY,OAAO;CACzB,MAAM,UAAU,OAAO;CACvB,MAAM,WAAW,OAAO;CACxB,MAAM,aAAa,OAAO;CAE1B,OAAO,MACJ,eAAe,EACd,UAAU;EACR,IAAI,MAAM;EACV,SAAS,MAAM;EACf,QAAQ,MAAM,OAAO;EACrB,QAAQ;EACR,UAAU,MAAM,OAAO;CACzB,EACF,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,YAAY,OAAO,UACtB,QAAQ,MAAM,MAAM,EACpB,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAW,OAAO;CAAM,CAAC,CAC1F,CACF;CAEF,MAAM,SAAS,OAAO,QACnB,MAAM;EACL,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,UAAU;EACnB,WAAW,MAAM,aAAA;EACjB,UAAU,UAAU;CACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,UAAU,OAAO,OAAO,IAC5B;EACE,YAAY,SAAS,WAAW,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,EAAE,KAClE,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;EACA,SAAS,WACN,UAAU;GACT,SAAS,UAAU;GACnB,aAAa,UAAU;GACvB,UAAU,UAAU;EACtB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;CACJ,GACA,EAAE,aAAa,YAAY,CAC7B;CAEA,IAAI,QAAQ,WAAW,WAAW,OAAO,QACvC,OAAO,OAAO,OAAO,KACnB,IAAI,wBAAwB;EAC1B,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,MAAM,gBAAgBA,MAAI,IAAI,QAAQ,QAAQ,UAAU,EAAE,KAAK,CAAC,OAAO,gBAAgB;EACrF;EACA;CACF,EAAE;CACF,OAAO,MACJ,sBAAsB;EACrB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,QAAQ;CACV,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,aAAa,OAAO,QAAQ,OAAO,UAAU,QAAQ,MAAM,YAAY,CAAC;CAC9E,OAAO,OAAO,MACX,kBAAkB;EACjB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,OAAO,QAAQ,QAAQ,SAAS,UAAU;EAC1C,SAAS,QAAQ,QAAQ,WAAW,UAAU;EAC9C,aAAa,MAAM;EACnB;EACA,YAAY,OAAO;CACrB,CAAC,EACA,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;AACJ,CAAC,EAAE,KACD,OAAO,SAAS,iCAAiC,GACjD,OAAO,OAAM,UACX,oBAAoB,OAAO,KAAK,EAAE,KAAK,OAAO,cAAc,OAAO,KAAK,KAAK,CAAC,CAAC,CACjF,CACF;AAQF,MAAa,+BAA2D,EACtE,QAAQ,wBACV"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yolk-sdk/knowledge",
|
|
3
|
-
"version": "0.1.0-canary.
|
|
3
|
+
"version": "0.1.0-canary.37",
|
|
4
4
|
"description": "Domain-free durable knowledge document, file, search, context, and tool contracts for Yolk agents.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -106,7 +106,7 @@
|
|
|
106
106
|
"dependencies": {
|
|
107
107
|
"effect": "4.0.0-beta.65",
|
|
108
108
|
"gpt-tokenizer": "^3.4.0",
|
|
109
|
-
"@yolk-sdk/agent": "^0.1.0-canary.
|
|
109
|
+
"@yolk-sdk/agent": "^0.1.0-canary.37"
|
|
110
110
|
},
|
|
111
111
|
"scripts": {
|
|
112
112
|
"build": "tsdown",
|
package/src/ingestion.ts
CHANGED
|
@@ -82,18 +82,16 @@ export const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>
|
|
|
82
82
|
|
|
83
83
|
const indexed = yield* Effect.all(
|
|
84
84
|
{
|
|
85
|
-
embeddings: embedder
|
|
86
|
-
.
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
)
|
|
96
|
-
),
|
|
85
|
+
embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(
|
|
86
|
+
Effect.mapError(
|
|
87
|
+
error =>
|
|
88
|
+
new KnowledgeIngestionError({
|
|
89
|
+
message: error.message,
|
|
90
|
+
stage: 'embed',
|
|
91
|
+
cause: error
|
|
92
|
+
})
|
|
93
|
+
)
|
|
94
|
+
),
|
|
97
95
|
summary: summarizer
|
|
98
96
|
.summarize({
|
|
99
97
|
content: extracted.content,
|