@yolk-sdk/knowledge 0.1.0-canary.13 → 0.1.0-canary.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -80
- package/dist/agent.d.mts +118 -19
- package/dist/agent.d.mts.map +1 -1
- package/dist/agent.mjs +256 -34
- package/dist/agent.mjs.map +1 -1
- package/dist/chunking.d.mts +13 -16
- package/dist/chunking.d.mts.map +1 -1
- package/dist/chunking.mjs +4 -4
- package/dist/chunking.mjs.map +1 -1
- package/dist/context.d.mts +4 -13
- package/dist/context.d.mts.map +1 -1
- package/dist/context.mjs +17 -13
- package/dist/context.mjs.map +1 -1
- package/dist/documents.d.mts +128 -78
- package/dist/documents.d.mts.map +1 -1
- package/dist/documents.mjs +89 -61
- package/dist/documents.mjs.map +1 -1
- package/dist/embeddings.d.mts +4 -7
- package/dist/embeddings.d.mts.map +1 -1
- package/dist/embeddings.mjs +1 -2
- package/dist/embeddings.mjs.map +1 -1
- package/dist/errors.d.mts +19 -22
- package/dist/errors.d.mts.map +1 -1
- package/dist/errors.mjs +12 -13
- package/dist/errors.mjs.map +1 -1
- package/dist/extraction.d.mts +4 -7
- package/dist/extraction.d.mts.map +1 -1
- package/dist/extraction.mjs +1 -2
- package/dist/extraction.mjs.map +1 -1
- package/dist/files.d.mts +19 -0
- package/dist/files.d.mts.map +1 -0
- package/dist/files.mjs +6 -0
- package/dist/files.mjs.map +1 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.mjs +4 -4
- package/dist/ingestion.d.mts +21 -23
- package/dist/ingestion.d.mts.map +1 -1
- package/dist/ingestion.mjs +12 -17
- package/dist/ingestion.mjs.map +1 -1
- package/dist/search.d.mts +13 -17
- package/dist/search.d.mts.map +1 -1
- package/dist/search.mjs +9 -12
- package/dist/search.mjs.map +1 -1
- package/dist/store.d.mts +92 -29
- package/dist/store.d.mts.map +1 -1
- package/dist/store.mjs +3 -3
- package/dist/store.mjs.map +1 -1
- package/dist/summarization.d.mts +6 -9
- package/dist/summarization.d.mts.map +1 -1
- package/dist/summarization.mjs +1 -2
- package/dist/summarization.mjs.map +1 -1
- package/package.json +11 -41
- package/src/agent.ts +393 -65
- package/src/chunking.ts +27 -8
- package/src/context.ts +21 -16
- package/src/documents.ts +140 -71
- package/src/errors.ts +4 -4
- package/src/extraction.ts +4 -3
- package/src/files.ts +24 -0
- package/src/index.ts +6 -3
- package/src/ingestion.ts +46 -27
- package/src/search.ts +90 -62
- package/src/store.ts +139 -25
- package/src/summarization.ts +4 -3
- package/dist/artifacts.d.mts +0 -37
- package/dist/artifacts.d.mts.map +0 -1
- package/dist/artifacts.mjs +0 -28
- package/dist/artifacts.mjs.map +0 -1
- package/dist/links.d.mts +0 -17
- package/dist/links.d.mts.map +0 -1
- package/dist/links.mjs +0 -24
- package/dist/links.mjs.map +0 -1
- package/dist/provenance.d.mts +0 -20
- package/dist/provenance.d.mts.map +0 -1
- package/dist/provenance.mjs +0 -26
- package/dist/provenance.mjs.map +0 -1
- package/dist/records.d.mts +0 -59
- package/dist/records.d.mts.map +0 -1
- package/dist/records.mjs +0 -64
- package/dist/records.mjs.map +0 -1
- package/dist/representations.d.mts +0 -36
- package/dist/representations.d.mts.map +0 -1
- package/dist/representations.mjs +0 -44
- package/dist/representations.mjs.map +0 -1
- package/dist/search-store.d.mts +0 -77
- package/dist/search-store.d.mts.map +0 -1
- package/dist/search-store.mjs +0 -7
- package/dist/search-store.mjs.map +0 -1
- package/dist/vector-store.d.mts +0 -3
- package/dist/vector-store.mjs +0 -2
- package/src/artifacts.ts +0 -48
- package/src/links.ts +0 -23
- package/src/provenance.ts +0 -25
- package/src/records.ts +0 -76
- package/src/representations.ts +0 -51
- package/src/search-store.ts +0 -98
- package/src/vector-store.ts +0 -6
package/dist/errors.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.mts","names":[],"sources":["../src/errors.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"errors.d.mts","names":[],"sources":["../src/errors.ts"],"mappings":";;;;qBAGa,8BAA8B;WAChC;WACA;;;;;qBAGE,iCAAiC;WACnC;WACA;;;;;qBAGE,+BAA+B;WACjC;WACA;;;;;qBAGE,gCAAgC;WAClC;WACA;;;;;qBAGE,oCAAoC;WACtC;WACA;;;;;qBAGE,gCAAgC;WAClC;WACA;WACA;;;;;qBAGE,6BAA6B;WAC/B;WACA;WACA;;;;;;qBAGE,4BAA4B;;;;;qBAQ5B,2BAA2B;;;;;qBAQ3B,8BAA8B"}
|
package/dist/errors.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Data } from "effect";
|
|
2
|
-
import * as Schema
|
|
3
|
-
|
|
2
|
+
import * as Schema from "effect/Schema";
|
|
3
|
+
// src/errors.ts
|
|
4
4
|
var SearchIndexStoreError = class extends Data.TaggedError("SearchIndexStoreError") {};
|
|
5
5
|
var KnowledgeExtractionError = class extends Data.TaggedError("KnowledgeExtractionError") {};
|
|
6
6
|
var KnowledgeChunkingError = class extends Data.TaggedError("KnowledgeChunkingError") {};
|
|
@@ -8,19 +8,18 @@ var KnowledgeEmbeddingError = class extends Data.TaggedError("KnowledgeEmbedding
|
|
|
8
8
|
var KnowledgeSummarizationError = class extends Data.TaggedError("KnowledgeSummarizationError") {};
|
|
9
9
|
var KnowledgeIngestionError = class extends Data.TaggedError("KnowledgeIngestionError") {};
|
|
10
10
|
var KnowledgeSearchError = class extends Data.TaggedError("KnowledgeSearchError") {};
|
|
11
|
-
var KnowledgeStoreError = class extends Schema
|
|
12
|
-
message: Schema
|
|
13
|
-
cause: Schema
|
|
11
|
+
var KnowledgeStoreError = class extends Schema.TaggedError()("KnowledgeStoreError", {
|
|
12
|
+
message: Schema.String,
|
|
13
|
+
cause: Schema.optional(Schema.Unknown)
|
|
14
14
|
}) {};
|
|
15
|
-
var
|
|
16
|
-
message: Schema
|
|
17
|
-
cause: Schema
|
|
15
|
+
var KnowledgeFileError = class extends Schema.TaggedError()("KnowledgeFileError", {
|
|
16
|
+
message: Schema.String,
|
|
17
|
+
cause: Schema.optional(Schema.Unknown)
|
|
18
18
|
}) {};
|
|
19
|
-
var KnowledgeContextError = class extends Schema
|
|
20
|
-
message: Schema
|
|
21
|
-
cause: Schema
|
|
19
|
+
var KnowledgeContextError = class extends Schema.TaggedError()("KnowledgeContextError", {
|
|
20
|
+
message: Schema.String,
|
|
21
|
+
cause: Schema.optional(Schema.Unknown)
|
|
22
22
|
}) {};
|
|
23
|
-
|
|
24
|
-
export { KnowledgeArtifactError, KnowledgeChunkingError, KnowledgeContextError, KnowledgeEmbeddingError, KnowledgeExtractionError, KnowledgeIngestionError, KnowledgeSearchError, KnowledgeStoreError, KnowledgeSummarizationError, SearchIndexStoreError };
|
|
23
|
+
export { KnowledgeChunkingError, KnowledgeContextError, KnowledgeEmbeddingError, KnowledgeExtractionError, KnowledgeFileError, KnowledgeIngestionError, KnowledgeSearchError, KnowledgeStoreError, KnowledgeSummarizationError, SearchIndexStoreError };
|
|
25
24
|
|
|
26
25
|
//# sourceMappingURL=errors.mjs.map
|
package/dist/errors.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.mjs","names":[
|
|
1
|
+
{"version":3,"file":"errors.mjs","names":[],"sources":["../src/errors.ts"],"sourcesContent":["import { Data } from 'effect'\nimport * as Schema from 'effect/Schema'\n\nexport class SearchIndexStoreError extends Data.TaggedError('SearchIndexStoreError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeExtractionError extends Data.TaggedError('KnowledgeExtractionError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeChunkingError extends Data.TaggedError('KnowledgeChunkingError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeEmbeddingError extends Data.TaggedError('KnowledgeEmbeddingError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeSummarizationError extends Data.TaggedError('KnowledgeSummarizationError')<{\n readonly message: string\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeIngestionError extends Data.TaggedError('KnowledgeIngestionError')<{\n readonly message: string\n readonly stage: 'store' | 'extract' | 'chunk' | 'embed' | 'summarize'\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeSearchError extends Data.TaggedError('KnowledgeSearchError')<{\n readonly message: string\n readonly stage: 'store' | 'embed'\n readonly cause?: unknown\n}> {}\n\nexport class KnowledgeStoreError extends Schema.TaggedError<KnowledgeStoreError>()(\n 'KnowledgeStoreError',\n {\n message: Schema.String,\n cause: Schema.optional(Schema.Unknown)\n }\n) {}\n\nexport class KnowledgeFileError extends Schema.TaggedError<KnowledgeFileError>()(\n 'KnowledgeFileError',\n {\n message: Schema.String,\n cause: Schema.optional(Schema.Unknown)\n }\n) {}\n\nexport class KnowledgeContextError extends Schema.TaggedError<KnowledgeContextError>()(\n 'KnowledgeContextError',\n {\n message: Schema.String,\n cause: Schema.optional(Schema.Unknown)\n }\n) {}\n"],"mappings":";;;AAGA,IAAa,wBAAb,cAA2C,KAAK,YAAY,uBAAuB,CAAC,CAGjF,CAAC;AAEJ,IAAa,2BAAb,cAA8C,KAAK,YAAY,0BAA0B,CAAC,CAGvF,CAAC;AAEJ,IAAa,yBAAb,cAA4C,KAAK,YAAY,wBAAwB,CAAC,CAGnF,CAAC;AAEJ,IAAa,0BAAb,cAA6C,KAAK,YAAY,yBAAyB,CAAC,CAGrF,CAAC;AAEJ,IAAa,8BAAb,cAAiD,KAAK,YAAY,6BAA6B,CAAC,CAG7F,CAAC;AAEJ,IAAa,0BAAb,cAA6C,KAAK,YAAY,yBAAyB,CAAC,CAIrF,CAAC;AAEJ,IAAa,uBAAb,cAA0C,KAAK,YAAY,sBAAsB,CAAC,CAI/E,CAAC;AAEJ,IAAa,sBAAb,cAAyC,OAAO,YAAiC,CAAC,CAChF,uBACA;CACE,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CACF,CAAC,CAAC,CAAC;AAEH,IAAa,qBAAb,cAAwC,OAAO,YAAgC,CAAC,CAC9E,sBACA;CACE,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CACF,CAAC,CAAC,CAAC;AAEH,IAAa,wBAAb,cAA2C,OAAO,YAAmC,CAAC,CACpF,yBACA;CACE,SAAS,OAAO;CAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AACvC,CACF,CAAC,CAAC,CAAC"}
|
package/dist/extraction.d.mts
CHANGED
|
@@ -1,19 +1,16 @@
|
|
|
1
1
|
import { ExtractedKnowledgeDocument, KnowledgeMetadata, KnowledgeSource } from "./documents.mjs";
|
|
2
2
|
import { KnowledgeExtractionError } from "./errors.mjs";
|
|
3
3
|
import { Context, Effect } from "effect";
|
|
4
|
-
|
|
5
|
-
//#region src/extraction.d.ts
|
|
6
|
-
type LoadedKnowledgeSource = {
|
|
4
|
+
export type LoadedKnowledgeSource = {
|
|
7
5
|
readonly source: KnowledgeSource;
|
|
8
6
|
readonly content: string | Uint8Array;
|
|
9
7
|
readonly mediaType?: string;
|
|
10
8
|
readonly metadata?: KnowledgeMetadata;
|
|
11
9
|
};
|
|
12
|
-
type KnowledgeExtractorApi = {
|
|
10
|
+
export type KnowledgeExtractorApi = {
|
|
13
11
|
readonly extract: (source: LoadedKnowledgeSource) => Effect.Effect<ExtractedKnowledgeDocument, KnowledgeExtractionError>;
|
|
14
12
|
};
|
|
15
13
|
declare const KnowledgeExtractor_base: Context.ServiceClass<KnowledgeExtractor, "@yolk-sdk/knowledge/KnowledgeExtractor", KnowledgeExtractorApi>;
|
|
16
|
-
declare class KnowledgeExtractor extends KnowledgeExtractor_base {}
|
|
17
|
-
|
|
18
|
-
export { KnowledgeExtractor, KnowledgeExtractorApi, LoadedKnowledgeSource };
|
|
14
|
+
export declare class KnowledgeExtractor extends KnowledgeExtractor_base {}
|
|
15
|
+
export {};
|
|
19
16
|
//# sourceMappingURL=extraction.d.mts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.d.mts","names":[],"sources":["../src/extraction.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"extraction.d.mts","names":[],"sources":["../src/extraction.ts"],"mappings":";;;YAKY;WACD,QAAQ;WACR,kBAAkB;WAClB;WACA,WAAW;;YAGV;WACD,UACP,QAAQ,0BACL,OAAO,OAAO,4BAA4B;;;qBAGpC,2BAA2B"}
|
package/dist/extraction.mjs
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Context } from "effect";
|
|
2
|
-
|
|
2
|
+
// src/extraction.ts
|
|
3
3
|
var KnowledgeExtractor = class extends Context.Service()("@yolk-sdk/knowledge/KnowledgeExtractor") {};
|
|
4
|
-
//#endregion
|
|
5
4
|
export { KnowledgeExtractor };
|
|
6
5
|
|
|
7
6
|
//# sourceMappingURL=extraction.mjs.map
|
package/dist/extraction.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extraction.mjs","names":[],"sources":["../src/extraction.ts"],"sourcesContent":["import { Context } from 'effect'\nimport type { Effect } from 'effect'\nimport type { ExtractedKnowledgeDocument, KnowledgeMetadata, KnowledgeSource } from './documents.ts'\nimport type { KnowledgeExtractionError } from './errors.ts'\n\nexport type LoadedKnowledgeSource = {\n readonly source: KnowledgeSource\n readonly content: string | Uint8Array\n readonly mediaType?: string\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeExtractorApi = {\n readonly extract: (\n source: LoadedKnowledgeSource\n ) => Effect.Effect<ExtractedKnowledgeDocument, KnowledgeExtractionError>\n}\n\nexport class KnowledgeExtractor extends Context.Service
|
|
1
|
+
{"version":3,"file":"extraction.mjs","names":[],"sources":["../src/extraction.ts"],"sourcesContent":["import { Context } from 'effect'\nimport type { Effect } from 'effect'\nimport type { ExtractedKnowledgeDocument, KnowledgeMetadata, KnowledgeSource } from './documents.ts'\nimport type { KnowledgeExtractionError } from './errors.ts'\n\nexport type LoadedKnowledgeSource = {\n readonly source: KnowledgeSource\n readonly content: string | Uint8Array\n readonly mediaType?: string\n readonly metadata?: KnowledgeMetadata\n}\n\nexport type KnowledgeExtractorApi = {\n readonly extract: (\n source: LoadedKnowledgeSource\n ) => Effect.Effect<ExtractedKnowledgeDocument, KnowledgeExtractionError>\n}\n\nexport class KnowledgeExtractor extends Context.Service<\n KnowledgeExtractor,\n KnowledgeExtractorApi\n>()('@yolk-sdk/knowledge/KnowledgeExtractor') {}\n"],"mappings":";;AAkBA,IAAa,qBAAb,cAAwC,QAAQ,QAG9C,CAAC,CAAC,wCAAwC,CAAC,CAAC,CAAC"}
|
package/dist/files.d.mts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { KnowledgeFileError } from "./errors.mjs";
|
|
2
|
+
import { Context, Effect } from "effect";
|
|
3
|
+
export type PutKnowledgeFileInput = {
|
|
4
|
+
readonly storageKey: string;
|
|
5
|
+
readonly mediaType?: string;
|
|
6
|
+
readonly bytes: Uint8Array;
|
|
7
|
+
};
|
|
8
|
+
export type GetKnowledgeFileInput = {
|
|
9
|
+
readonly storageKey: string;
|
|
10
|
+
};
|
|
11
|
+
export type KnowledgeFileBlobStoreApi = {
|
|
12
|
+
readonly putFile: (input: PutKnowledgeFileInput) => Effect.Effect<void, KnowledgeFileError>;
|
|
13
|
+
readonly getFile: (input: GetKnowledgeFileInput) => Effect.Effect<Uint8Array, KnowledgeFileError>;
|
|
14
|
+
readonly deleteFile: (input: GetKnowledgeFileInput) => Effect.Effect<void, KnowledgeFileError>;
|
|
15
|
+
};
|
|
16
|
+
declare const KnowledgeFileBlobStore_base: Context.ServiceClass<KnowledgeFileBlobStore, "@yolk-sdk/knowledge/KnowledgeFileBlobStore", KnowledgeFileBlobStoreApi>;
|
|
17
|
+
export declare class KnowledgeFileBlobStore extends KnowledgeFileBlobStore_base {}
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=files.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.d.mts","names":[],"sources":["../src/files.ts"],"mappings":";;YAIY;WACD;WACA;WACA,OAAO;;YAGN;WACD;;YAGC;WACD,UAAU,OAAO,0BAA0B,OAAO,aAAa;WAC/D,UAAU,OAAO,0BAA0B,OAAO,OAAO,YAAY;WACrE,aAAa,OAAO,0BAA0B,OAAO,aAAa;;;qBAGhE,+BAA+B"}
|
package/dist/files.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"files.mjs","names":[],"sources":["../src/files.ts"],"sourcesContent":["import { Context } from 'effect'\nimport type { Effect } from 'effect'\nimport type { KnowledgeFileError } from './errors.ts'\n\nexport type PutKnowledgeFileInput = {\n readonly storageKey: string\n readonly mediaType?: string\n readonly bytes: Uint8Array\n}\n\nexport type GetKnowledgeFileInput = {\n readonly storageKey: string\n}\n\nexport type KnowledgeFileBlobStoreApi = {\n readonly putFile: (input: PutKnowledgeFileInput) => Effect.Effect<void, KnowledgeFileError>\n readonly getFile: (input: GetKnowledgeFileInput) => Effect.Effect<Uint8Array, KnowledgeFileError>\n readonly deleteFile: (input: GetKnowledgeFileInput) => Effect.Effect<void, KnowledgeFileError>\n}\n\nexport class KnowledgeFileBlobStore extends Context.Service<\n KnowledgeFileBlobStore,\n KnowledgeFileBlobStoreApi\n>()('@yolk-sdk/knowledge/KnowledgeFileBlobStore') {}\n"],"mappings":";;AAoBA,IAAa,yBAAb,cAA4C,QAAQ,QAGlD,CAAC,CAAC,4CAA4C,CAAC,CAAC,CAAC"}
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { KnowledgeArtifactError, KnowledgeContextError, KnowledgeStoreError } from "./errors.mjs";
|
|
2
|
-
import { KnowledgeArtifactStore } from "./artifacts.mjs";
|
|
3
1
|
import { buildKnowledgeContext } from "./context.mjs";
|
|
4
|
-
import { KnowledgeStore } from "./store.mjs";
|
|
5
|
-
|
|
2
|
+
import { KnowledgeStore, SearchIndexStore } from "./store.mjs";
|
|
3
|
+
import { KnowledgeFileBlobStore } from "./files.mjs";
|
|
4
|
+
import { KnowledgeContextError, KnowledgeFileError, KnowledgeStoreError } from "./errors.mjs";
|
|
5
|
+
export { KnowledgeContextError, KnowledgeFileBlobStore, KnowledgeFileError, KnowledgeStore, KnowledgeStoreError, SearchIndexStore, buildKnowledgeContext };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { KnowledgeArtifactError, KnowledgeContextError, KnowledgeStoreError } from "./errors.mjs";
|
|
2
|
-
import { KnowledgeArtifactStore } from "./artifacts.mjs";
|
|
3
1
|
import { buildKnowledgeContext } from "./context.mjs";
|
|
4
|
-
import { KnowledgeStore } from "./store.mjs";
|
|
5
|
-
|
|
2
|
+
import { KnowledgeStore, SearchIndexStore } from "./store.mjs";
|
|
3
|
+
import { KnowledgeFileBlobStore } from "./files.mjs";
|
|
4
|
+
import { KnowledgeContextError, KnowledgeFileError, KnowledgeStoreError } from "./errors.mjs";
|
|
5
|
+
export { KnowledgeContextError, KnowledgeFileBlobStore, KnowledgeFileError, KnowledgeStore, KnowledgeStoreError, SearchIndexStore, buildKnowledgeContext };
|
package/dist/ingestion.d.mts
CHANGED
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
import { KnowledgeIngestionError } from "./errors.mjs";
|
|
2
|
-
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
3
|
-
import { SearchIndexStore } from "./search-store.mjs";
|
|
4
1
|
import { KnowledgeChunker } from "./chunking.mjs";
|
|
2
|
+
import { KnowledgeDocumentId, KnowledgeScopeId } from "./documents.mjs";
|
|
3
|
+
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
5
4
|
import { KnowledgeExtractor, LoadedKnowledgeSource } from "./extraction.mjs";
|
|
6
5
|
import { KnowledgeSummarizer } from "./summarization.mjs";
|
|
6
|
+
import { KnowledgeIngestionError } from "./errors.mjs";
|
|
7
|
+
import { SearchIndexStore } from "./store.mjs";
|
|
7
8
|
import { Effect } from "effect";
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
readonly collectionId: string;
|
|
12
|
-
readonly documentId: string;
|
|
9
|
+
export type IngestKnowledgeDocumentInput = {
|
|
10
|
+
readonly scopeId: KnowledgeScopeId;
|
|
11
|
+
readonly documentId: KnowledgeDocumentId;
|
|
13
12
|
readonly source: LoadedKnowledgeSource;
|
|
14
13
|
readonly contentHash?: string;
|
|
14
|
+
readonly maxTokens?: number;
|
|
15
15
|
};
|
|
16
|
-
declare const ingestKnowledgeDocument: (input: IngestKnowledgeDocumentInput) => Effect.Effect<{
|
|
17
|
-
readonly id: string
|
|
18
|
-
readonly
|
|
16
|
+
export declare const ingestKnowledgeDocument: (input: IngestKnowledgeDocumentInput) => Effect.Effect<{
|
|
17
|
+
readonly id: string & import("effect/Brand").Brand<"KnowledgeDocumentId">;
|
|
18
|
+
readonly scopeId: string & import("effect/Brand").Brand<"KnowledgeScopeId">;
|
|
19
19
|
readonly source: {
|
|
20
20
|
readonly _tag: "File";
|
|
21
21
|
readonly ref: string;
|
|
22
|
-
readonly mediaType?: string | undefined;
|
|
23
22
|
readonly name?: string | undefined;
|
|
23
|
+
readonly mediaType?: string | undefined;
|
|
24
24
|
} | {
|
|
25
25
|
readonly _tag: "Url";
|
|
26
26
|
readonly url: string;
|
|
@@ -28,21 +28,19 @@ declare const ingestKnowledgeDocument: (input: IngestKnowledgeDocumentInput) =>
|
|
|
28
28
|
readonly _tag: "Text";
|
|
29
29
|
readonly label?: string | undefined;
|
|
30
30
|
};
|
|
31
|
-
readonly status: "
|
|
32
|
-
readonly metadata?: {
|
|
33
|
-
readonly [x: string]: unknown;
|
|
34
|
-
} | undefined;
|
|
35
|
-
readonly tokenCount?: number | undefined;
|
|
36
|
-
readonly summary?: string | undefined;
|
|
31
|
+
readonly status: "error" | "processing" | "ready";
|
|
37
32
|
readonly title?: string | undefined;
|
|
33
|
+
readonly summary?: string | undefined;
|
|
38
34
|
readonly errorMessage?: string | undefined;
|
|
39
35
|
readonly contentHash?: string | undefined;
|
|
36
|
+
readonly tokenCount?: number | undefined;
|
|
40
37
|
readonly chunkCount?: number | undefined;
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
readonly metadata?: {
|
|
39
|
+
readonly [x: string]: unknown;
|
|
40
|
+
} | undefined;
|
|
41
|
+
}, KnowledgeIngestionError, KnowledgeChunker | KnowledgeEmbedder | KnowledgeExtractor | KnowledgeSummarizer | SearchIndexStore>;
|
|
42
|
+
export type KnowledgeIngestionPipeline = {
|
|
43
43
|
readonly ingest: (input: IngestKnowledgeDocumentInput) => ReturnType<typeof ingestKnowledgeDocument>;
|
|
44
44
|
};
|
|
45
|
-
declare const makeIngestionPipeline: () => KnowledgeIngestionPipeline;
|
|
46
|
-
//#endregion
|
|
47
|
-
export { IngestKnowledgeDocumentInput, KnowledgeIngestionPipeline, ingestKnowledgeDocument, makeIngestionPipeline };
|
|
45
|
+
export declare const makeIngestionPipeline: () => KnowledgeIngestionPipeline;
|
|
48
46
|
//# sourceMappingURL=ingestion.d.mts.map
|
package/dist/ingestion.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion.d.mts","names":[],"sources":["../src/ingestion.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"ingestion.d.mts","names":[],"sources":["../src/ingestion.ts"],"mappings":";;;;;;;;YAWY;WACD,SAAS;WACT,YAAY;WACZ,QAAQ;WACR;WACA;;qBAaE,0BAAuB,OAAW,iCAA4B,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAwIxE,yBAAA,mBAAA,oBAAA,qBAAA,sBAAA;YAES;WACD,SACP,OAAO,iCACJ,kBAAkB;;qBAGZ,6BAA4B"}
|
package/dist/ingestion.mjs
CHANGED
|
@@ -1,21 +1,22 @@
|
|
|
1
|
-
import { KnowledgeIngestionError } from "./errors.mjs";
|
|
2
|
-
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
3
|
-
import { SearchIndexStore } from "./search-store.mjs";
|
|
4
1
|
import { KnowledgeChunker } from "./chunking.mjs";
|
|
2
|
+
import "./documents.mjs";
|
|
3
|
+
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
5
4
|
import { KnowledgeExtractor } from "./extraction.mjs";
|
|
6
5
|
import { KnowledgeSummarizer } from "./summarization.mjs";
|
|
6
|
+
import { KnowledgeIngestionError } from "./errors.mjs";
|
|
7
|
+
import { SearchIndexStore } from "./store.mjs";
|
|
7
8
|
import { Array, Effect } from "effect";
|
|
8
|
-
|
|
9
|
+
// src/ingestion.ts
|
|
9
10
|
const markErrorBestEffort = (input, error) => Effect.gen(function* () {
|
|
10
11
|
yield* (yield* SearchIndexStore).markDocumentError({
|
|
11
|
-
|
|
12
|
+
scopeId: input.scopeId,
|
|
12
13
|
documentId: input.documentId,
|
|
13
14
|
message: error.message
|
|
14
15
|
});
|
|
15
16
|
}).pipe(Effect.catch(() => Effect.void));
|
|
16
17
|
const ingestKnowledgeDocument = (input) => Effect.gen(function* () {
|
|
17
18
|
yield* Effect.annotateCurrentSpan({
|
|
18
|
-
"knowledge_search.
|
|
19
|
+
"knowledge_search.scope_id": input.scopeId,
|
|
19
20
|
"knowledge_search.document_id": input.documentId,
|
|
20
21
|
"knowledge_search.source_type": input.source.source._tag
|
|
21
22
|
});
|
|
@@ -24,14 +25,9 @@ const ingestKnowledgeDocument = (input) => Effect.gen(function* () {
|
|
|
24
25
|
const chunker = yield* KnowledgeChunker;
|
|
25
26
|
const embedder = yield* KnowledgeEmbedder;
|
|
26
27
|
const summarizer = yield* KnowledgeSummarizer;
|
|
27
|
-
const collection = yield* store.getSet(input.collectionId).pipe(Effect.mapError((error) => new KnowledgeIngestionError({
|
|
28
|
-
message: error.message,
|
|
29
|
-
stage: "store",
|
|
30
|
-
cause: error
|
|
31
|
-
})));
|
|
32
28
|
yield* store.upsertDocument({ document: {
|
|
33
29
|
id: input.documentId,
|
|
34
|
-
|
|
30
|
+
scopeId: input.scopeId,
|
|
35
31
|
source: input.source.source,
|
|
36
32
|
status: "processing",
|
|
37
33
|
metadata: input.source.metadata
|
|
@@ -46,10 +42,10 @@ const ingestKnowledgeDocument = (input) => Effect.gen(function* () {
|
|
|
46
42
|
cause: error
|
|
47
43
|
})));
|
|
48
44
|
const chunks = yield* chunker.chunk({
|
|
49
|
-
|
|
45
|
+
scopeId: input.scopeId,
|
|
50
46
|
documentId: input.documentId,
|
|
51
47
|
content: extracted.content,
|
|
52
|
-
maxTokens:
|
|
48
|
+
maxTokens: input.maxTokens ?? 512,
|
|
53
49
|
metadata: extracted.metadata
|
|
54
50
|
}).pipe(Effect.mapError((error) => new KnowledgeIngestionError({
|
|
55
51
|
message: error.message,
|
|
@@ -81,7 +77,7 @@ const ingestKnowledgeDocument = (input) => Effect.gen(function* () {
|
|
|
81
77
|
embedding
|
|
82
78
|
}));
|
|
83
79
|
yield* store.replaceDocumentChunks({
|
|
84
|
-
|
|
80
|
+
scopeId: input.scopeId,
|
|
85
81
|
documentId: input.documentId,
|
|
86
82
|
chunks: indexedChunks
|
|
87
83
|
}).pipe(Effect.mapError((error) => new KnowledgeIngestionError({
|
|
@@ -91,7 +87,7 @@ const ingestKnowledgeDocument = (input) => Effect.gen(function* () {
|
|
|
91
87
|
})));
|
|
92
88
|
const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0);
|
|
93
89
|
return yield* store.markDocumentReady({
|
|
94
|
-
|
|
90
|
+
scopeId: input.scopeId,
|
|
95
91
|
documentId: input.documentId,
|
|
96
92
|
title: indexed.summary.title ?? extracted.title,
|
|
97
93
|
summary: indexed.summary.summary ?? extracted.summary,
|
|
@@ -105,7 +101,6 @@ const ingestKnowledgeDocument = (input) => Effect.gen(function* () {
|
|
|
105
101
|
})));
|
|
106
102
|
}).pipe(Effect.withSpan("knowledge_search.ingestDocument"), Effect.catch((error) => markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))));
|
|
107
103
|
const makeIngestionPipeline = () => ({ ingest: ingestKnowledgeDocument });
|
|
108
|
-
//#endregion
|
|
109
104
|
export { ingestKnowledgeDocument, makeIngestionPipeline };
|
|
110
105
|
|
|
111
106
|
//# sourceMappingURL=ingestion.mjs.map
|
package/dist/ingestion.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './
|
|
1
|
+
{"version":3,"file":"ingestion.mjs","names":["Arr"],"sources":["../src/ingestion.ts"],"sourcesContent":["import { Array as Arr, Effect } from 'effect'\nimport { KnowledgeChunker } from './chunking.ts'\nimport { defaultKnowledgeChunkMaxTokens } from './documents.ts'\nimport type { KnowledgeDocumentId, KnowledgeScopeId } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeExtractor } from './extraction.ts'\nimport { KnowledgeSummarizer } from './summarization.ts'\nimport type { LoadedKnowledgeSource } from './extraction.ts'\nimport { KnowledgeIngestionError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type IngestKnowledgeDocumentInput = {\n readonly scopeId: KnowledgeScopeId\n readonly documentId: KnowledgeDocumentId\n readonly source: LoadedKnowledgeSource\n readonly contentHash?: string\n readonly maxTokens?: number\n}\n\nconst markErrorBestEffort = (input: IngestKnowledgeDocumentInput, error: KnowledgeIngestionError) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n yield* store.markDocumentError({\n scopeId: input.scopeId,\n documentId: input.documentId,\n message: error.message\n })\n }).pipe(Effect.catch(() => Effect.void))\n\nexport const ingestKnowledgeDocument = (input: IngestKnowledgeDocumentInput) =>\n Effect.gen(function* () {\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.scope_id': input.scopeId,\n 'knowledge_search.document_id': input.documentId,\n 'knowledge_search.source_type': input.source.source._tag\n })\n const store = yield* SearchIndexStore\n const extractor = yield* KnowledgeExtractor\n const chunker = yield* KnowledgeChunker\n const embedder = yield* KnowledgeEmbedder\n const summarizer = yield* KnowledgeSummarizer\n\n yield* store\n .upsertDocument({\n document: {\n id: input.documentId,\n scopeId: input.scopeId,\n source: input.source.source,\n status: 'processing',\n metadata: input.source.metadata\n }\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const extracted = yield* extractor\n .extract(input.source)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'extract', cause: error })\n )\n )\n\n const chunks = yield* chunker\n .chunk({\n scopeId: input.scopeId,\n documentId: input.documentId,\n content: extracted.content,\n maxTokens: input.maxTokens ?? defaultKnowledgeChunkMaxTokens,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'chunk', cause: error })\n )\n )\n\n const indexed = yield* Effect.all(\n {\n embeddings: embedder.embedTexts(chunks.map(chunk => chunk.content)).pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'embed',\n cause: error\n })\n )\n ),\n summary: summarizer\n .summarize({\n content: extracted.content,\n sourceTitle: extracted.title,\n metadata: extracted.metadata\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({\n message: error.message,\n stage: 'summarize',\n cause: error\n })\n )\n )\n },\n { concurrency: 'unbounded' }\n )\n\n if (indexed.embeddings.length !== chunks.length) {\n return yield* Effect.fail(\n new KnowledgeIngestionError({\n message: 'Embedding count did not match chunk count',\n stage: 'embed'\n })\n )\n }\n\n const indexedChunks = Arr.zip(chunks, indexed.embeddings).map(([chunk, embedding]) => ({\n chunk,\n embedding\n }))\n\n yield* store\n .replaceDocumentChunks({\n scopeId: input.scopeId,\n documentId: input.documentId,\n chunks: indexedChunks\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n\n const tokenCount = chunks.reduce((total, chunk) => total + chunk.tokenCount, 0)\n\n return yield* store\n .markDocumentReady({\n scopeId: input.scopeId,\n documentId: input.documentId,\n title: indexed.summary.title ?? extracted.title,\n summary: indexed.summary.summary ?? extracted.summary,\n contentHash: input.contentHash,\n tokenCount,\n chunkCount: chunks.length\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeIngestionError({ message: error.message, stage: 'store', cause: error })\n )\n )\n }).pipe(\n Effect.withSpan('knowledge_search.ingestDocument'),\n Effect.catch(error =>\n markErrorBestEffort(input, error).pipe(Effect.flatMap(() => Effect.fail(error)))\n )\n )\n\nexport type KnowledgeIngestionPipeline = {\n readonly ingest: (\n input: IngestKnowledgeDocumentInput\n ) => ReturnType<typeof ingestKnowledgeDocument>\n}\n\nexport const makeIngestionPipeline = (): KnowledgeIngestionPipeline => ({\n ingest: ingestKnowledgeDocument\n})\n"],"mappings":";;;;;;;;;AAmBA,MAAM,uBAAuB,OAAqC,UAChE,OAAO,IAAI,aAAa;CAEtB,QAAO,OADc,iBAAA,CACR,kBAAkB;EAC7B,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,MAAM;CACjB,CAAC;AACH,CAAC,CAAC,CAAC,KAAK,OAAO,YAAY,OAAO,IAAI,CAAC;AAEzC,MAAa,2BAA2B,UACtC,OAAO,IAAI,aAAa;CACtB,OAAO,OAAO,oBAAoB;EAChC,6BAA6B,MAAM;EACnC,gCAAgC,MAAM;EACtC,gCAAgC,MAAM,OAAO,OAAO;CACtD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,YAAY,OAAO;CACzB,MAAM,UAAU,OAAO;CACvB,MAAM,WAAW,OAAO;CACxB,MAAM,aAAa,OAAO;CAE1B,OAAO,MACJ,eAAe,EACd,UAAU;EACR,IAAI,MAAM;EACV,SAAS,MAAM;EACf,QAAQ,MAAM,OAAO;EACrB,QAAQ;EACR,UAAU,MAAM,OAAO;CACzB,EACF,CAAC,CAAC,CACD,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,YAAY,OAAO,UACtB,QAAQ,MAAM,MAAM,CAAC,CACrB,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAW,OAAO;CAAM,CAAC,CAC1F,CACF;CAEF,MAAM,SAAS,OAAO,QACnB,MAAM;EACL,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,SAAS,UAAU;EACnB,WAAW,MAAM,aAAA;EACjB,UAAU,UAAU;CACtB,CAAC,CAAC,CACD,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,UAAU,OAAO,OAAO,IAC5B;EACE,YAAY,SAAS,WAAW,OAAO,KAAI,UAAS,MAAM,OAAO,CAAC,CAAC,CAAC,KAClE,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;EACA,SAAS,WACN,UAAU;GACT,SAAS,UAAU;GACnB,aAAa,UAAU;GACvB,UAAU,UAAU;EACtB,CAAC,CAAC,CACD,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;GAC1B,SAAS,MAAM;GACf,OAAO;GACP,OAAO;EACT,CAAC,CACL,CACF;CACJ,GACA,EAAE,aAAa,YAAY,CAC7B;CAEA,IAAI,QAAQ,WAAW,WAAW,OAAO,QACvC,OAAO,OAAO,OAAO,KACnB,IAAI,wBAAwB;EAC1B,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,MAAM,gBAAgBA,MAAI,IAAI,QAAQ,QAAQ,UAAU,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB;EACrF;EACA;CACF,EAAE;CAEF,OAAO,MACJ,sBAAsB;EACrB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,QAAQ;CACV,CAAC,CAAC,CACD,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;CAEF,MAAM,aAAa,OAAO,QAAQ,OAAO,UAAU,QAAQ,MAAM,YAAY,CAAC;CAE9E,OAAO,OAAO,MACX,kBAAkB;EACjB,SAAS,MAAM;EACf,YAAY,MAAM;EAClB,OAAO,QAAQ,QAAQ,SAAS,UAAU;EAC1C,SAAS,QAAQ,QAAQ,WAAW,UAAU;EAC9C,aAAa,MAAM;EACnB;EACA,YAAY,OAAO;CACrB,CAAC,CAAC,CACD,KACC,OAAO,UACL,UACE,IAAI,wBAAwB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACxF,CACF;AACJ,CAAC,CAAC,CAAC,KACD,OAAO,SAAS,iCAAiC,GACjD,OAAO,OAAM,UACX,oBAAoB,OAAO,KAAK,CAAC,CAAC,KAAK,OAAO,cAAc,OAAO,KAAK,KAAK,CAAC,CAAC,CACjF,CACF;AAQF,MAAa,+BAA2D,EACtE,QAAQ,wBACV"}
|
package/dist/search.d.mts
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { KnowledgeSearchError } from "./errors.mjs";
|
|
1
|
+
import { IndexedKnowledgeDocument, KnowledgeChunk, KnowledgeSearchScope } from "./documents.mjs";
|
|
3
2
|
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
4
|
-
import {
|
|
3
|
+
import { KnowledgeSearchError } from "./errors.mjs";
|
|
4
|
+
import { SearchIndexStore } from "./store.mjs";
|
|
5
5
|
import { Effect } from "effect";
|
|
6
|
-
|
|
7
|
-
//#region src/search.d.ts
|
|
8
|
-
type KnowledgeSearchInput = {
|
|
6
|
+
export type KnowledgeSearchInput = {
|
|
9
7
|
readonly scope: KnowledgeSearchScope;
|
|
10
8
|
readonly query: string;
|
|
11
9
|
readonly limit?: number;
|
|
@@ -15,35 +13,33 @@ type KnowledgeSearchInput = {
|
|
|
15
13
|
readonly vectorLimit?: number;
|
|
16
14
|
readonly textLimit?: number;
|
|
17
15
|
};
|
|
18
|
-
type KnowledgeSearchMode = 'vector' | 'hybrid';
|
|
19
|
-
type KnowledgeSearchScores = {
|
|
16
|
+
export type KnowledgeSearchMode = 'vector' | 'hybrid';
|
|
17
|
+
export type KnowledgeSearchScores = {
|
|
20
18
|
readonly vector?: number;
|
|
21
19
|
readonly text?: number;
|
|
22
20
|
readonly fused?: number;
|
|
23
21
|
};
|
|
24
|
-
type KnowledgeSearchResult = {
|
|
22
|
+
export type KnowledgeSearchResult = {
|
|
25
23
|
readonly chunk: KnowledgeChunk;
|
|
26
24
|
readonly score: number;
|
|
27
|
-
readonly document:
|
|
25
|
+
readonly document: IndexedKnowledgeDocument;
|
|
28
26
|
readonly context?: ReadonlyArray<KnowledgeChunk>;
|
|
29
27
|
readonly scores?: KnowledgeSearchScores;
|
|
30
28
|
};
|
|
31
|
-
type KnowledgeSearchContext = {
|
|
29
|
+
export type KnowledgeSearchContext = {
|
|
32
30
|
readonly query: string;
|
|
33
31
|
readonly results: ReadonlyArray<KnowledgeSearchResult>;
|
|
34
32
|
readonly text: string;
|
|
35
33
|
};
|
|
36
|
-
type KnowledgeSearcher = {
|
|
34
|
+
export type KnowledgeSearcher = {
|
|
37
35
|
readonly search: (input: KnowledgeSearchInput) => Effect.Effect<ReadonlyArray<KnowledgeSearchResult>, KnowledgeSearchError>;
|
|
38
36
|
};
|
|
39
|
-
declare const packKnowledgeSearchContext: (query: string, results: ReadonlyArray<KnowledgeSearchResult>) => KnowledgeSearchContext;
|
|
40
|
-
declare const fuseKnowledgeSearchResults: (input: {
|
|
37
|
+
export declare const packKnowledgeSearchContext: (query: string, results: ReadonlyArray<KnowledgeSearchResult>) => KnowledgeSearchContext;
|
|
38
|
+
export declare const fuseKnowledgeSearchResults: (input: {
|
|
41
39
|
readonly vectorResults: ReadonlyArray<KnowledgeSearchResult>;
|
|
42
40
|
readonly textResults: ReadonlyArray<KnowledgeSearchResult>;
|
|
43
41
|
readonly limit: number;
|
|
44
42
|
readonly rankConstant?: number;
|
|
45
43
|
}) => ReadonlyArray<KnowledgeSearchResult>;
|
|
46
|
-
declare const searchKnowledge: (input: KnowledgeSearchInput) => Effect.Effect<readonly KnowledgeSearchResult[], KnowledgeSearchError, KnowledgeEmbedder | SearchIndexStore>;
|
|
47
|
-
//#endregion
|
|
48
|
-
export { KnowledgeSearchContext, KnowledgeSearchInput, KnowledgeSearchMode, KnowledgeSearchResult, KnowledgeSearchScores, KnowledgeSearcher, fuseKnowledgeSearchResults, packKnowledgeSearchContext, searchKnowledge };
|
|
44
|
+
export declare const searchKnowledge: (input: KnowledgeSearchInput) => Effect.Effect<readonly KnowledgeSearchResult[], KnowledgeSearchError, KnowledgeEmbedder | SearchIndexStore>;
|
|
49
45
|
//# sourceMappingURL=search.d.mts.map
|
package/dist/search.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.mts","names":[],"sources":["../src/search.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"search.d.mts","names":[],"sources":["../src/search.ts"],"mappings":";;;;;YAMY;WACD,OAAO;WACP;WACA;WACA;WACA;WACA,OAAO;WACP;WACA;;YAGC;YAEA;WACD;WACA;WACA;;YAGC;WACD,OAAO;WACP;WACA,UAAU;WACV,UAAU,cAAc;WACxB,SAAS;;YAGR;WACD;WACA,SAAS,cAAc;WACvB;;YAGC;WACD,SACP,OAAO,yBACJ,OAAO,OAAO,cAAc,wBAAwB;;qBAM9C,6BAA0B,eACxB,SACJ,cAAc,2BACtB;qBAkBU,6BAA0B;WAC5B,eAAe,cAAc;WAC7B,aAAa,cAAc;WAC3B;WACA;MACP,cAAc;qBA8JL,kBAAe,OAAW,yBAAoB,OAAA,gBAAA,yBAAA,sBAAA,oBAAA"}
|
package/dist/search.mjs
CHANGED
|
@@ -1,20 +1,18 @@
|
|
|
1
|
-
import { KnowledgeSearchError } from "./errors.mjs";
|
|
2
1
|
import { KnowledgeEmbedder } from "./embeddings.mjs";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
|
|
2
|
+
import { KnowledgeSearchError } from "./errors.mjs";
|
|
3
|
+
import { SearchIndexStore } from "./store.mjs";
|
|
4
|
+
import { Effect, Match } from "effect";
|
|
5
|
+
// src/search.ts
|
|
6
6
|
const packedResultText = (result) => result.context?.map((chunk) => chunk.content).join("\n\n") ?? result.chunk.content;
|
|
7
7
|
const packKnowledgeSearchContext = (query, results) => ({
|
|
8
8
|
query,
|
|
9
9
|
results,
|
|
10
10
|
text: results.map(packedResultText).join("\n\n")
|
|
11
11
|
});
|
|
12
|
-
const scopeIds = (scope) => {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
17
|
-
};
|
|
12
|
+
const scopeIds = (scope) => Match.value(scope).pipe(Match.tagsExhaustive({
|
|
13
|
+
KnowledgeScope: ({ id }) => [id],
|
|
14
|
+
KnowledgeScopes: ({ ids }) => ids
|
|
15
|
+
}));
|
|
18
16
|
const defaultHybridCandidateLimit = (limit) => Math.max(limit * 5, 40);
|
|
19
17
|
const reciprocalRankFusionScore = (rank, rankConstant) => 1 / (rankConstant + rank);
|
|
20
18
|
const fuseKnowledgeSearchResults = (input) => {
|
|
@@ -160,7 +158,7 @@ const searchKnowledge = (input) => Effect.gen(function* () {
|
|
|
160
158
|
});
|
|
161
159
|
if (valid.contextChunks === 0) return results;
|
|
162
160
|
return yield* Effect.forEach(results, (result) => store.getContextChunks({
|
|
163
|
-
|
|
161
|
+
scopeId: result.chunk.scopeId,
|
|
164
162
|
documentId: result.chunk.documentId,
|
|
165
163
|
position: result.chunk.position,
|
|
166
164
|
contextChunks: valid.contextChunks
|
|
@@ -173,7 +171,6 @@ const searchKnowledge = (input) => Effect.gen(function* () {
|
|
|
173
171
|
cause: error
|
|
174
172
|
}))));
|
|
175
173
|
}).pipe(Effect.withSpan("knowledge_search.search"));
|
|
176
|
-
//#endregion
|
|
177
174
|
export { fuseKnowledgeSearchResults, packKnowledgeSearchContext, searchKnowledge };
|
|
178
175
|
|
|
179
176
|
//# sourceMappingURL=search.mjs.map
|
package/dist/search.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.mjs","names":[],"sources":["../src/search.ts"],"sourcesContent":["import { Effect } from 'effect'\nimport type { KnowledgeChunk, KnowledgeDocument, KnowledgeSearchScope } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeSearchError } from './errors.ts'\nimport { SearchIndexStore } from './search-store.ts'\n\nexport type KnowledgeSearchInput = {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n readonly mode?: KnowledgeSearchMode\n readonly vectorLimit?: number\n readonly textLimit?: number\n}\n\nexport type KnowledgeSearchMode = 'vector' | 'hybrid'\n\nexport type KnowledgeSearchScores = {\n readonly vector?: number\n readonly text?: number\n readonly fused?: number\n}\n\nexport type KnowledgeSearchResult = {\n readonly chunk: KnowledgeChunk\n readonly score: number\n readonly document: KnowledgeDocument\n readonly context?: ReadonlyArray<KnowledgeChunk>\n readonly scores?: KnowledgeSearchScores\n}\n\nexport type KnowledgeSearchContext = {\n readonly query: string\n readonly results: ReadonlyArray<KnowledgeSearchResult>\n readonly text: string\n}\n\nexport type KnowledgeSearcher = {\n readonly search: (\n input: KnowledgeSearchInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeSearchResult>, KnowledgeSearchError>\n}\n\nconst packedResultText = (result: KnowledgeSearchResult) =>\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.chunk.content\n\nexport const packKnowledgeSearchContext = (query: string, results: ReadonlyArray<KnowledgeSearchResult>): KnowledgeSearchContext =>\n ({\n query,\n results,\n text: results.map(packedResultText).join('\\n\\n')\n })\n\nconst scopeIds = (scope: KnowledgeSearchScope): ReadonlyArray<string> => {\n switch (scope._tag) {\n case 'KnowledgeCollection':\n return [scope.id]\n case 'KnowledgeCollections':\n return scope.ids\n }\n}\n\nconst defaultHybridCandidateLimit = (limit: number) => Math.max(limit * 5, 40)\n\nconst reciprocalRankFusionScore = (rank: number, rankConstant: number) => 1 / (rankConstant + rank)\n\nexport const fuseKnowledgeSearchResults = (input: {\n readonly vectorResults: ReadonlyArray<KnowledgeSearchResult>\n readonly textResults: ReadonlyArray<KnowledgeSearchResult>\n readonly limit: number\n readonly rankConstant?: number\n}): ReadonlyArray<KnowledgeSearchResult> => {\n const rankConstant = input.rankConstant ?? 60\n const fused = new Map<\n string,\n {\n readonly result: KnowledgeSearchResult\n readonly vector?: number\n readonly text?: number\n readonly fused: number\n }\n >()\n\n const addResults = (results: ReadonlyArray<KnowledgeSearchResult>, kind: 'vector' | 'text') => {\n results.forEach((result, index) => {\n const rank = index + 1\n const previous = fused.get(result.chunk.id)\n const nextFused = (previous?.fused ?? 0) + reciprocalRankFusionScore(rank, rankConstant)\n fused.set(result.chunk.id, {\n result: previous?.result ?? result,\n vector: kind === 'vector' ? result.score : previous?.vector,\n text: kind === 'text' ? result.score : previous?.text,\n fused: nextFused\n })\n })\n }\n\n addResults(input.vectorResults, 'vector')\n addResults(input.textResults, 'text')\n\n return Array.from(fused.values())\n .sort((left, right) => right.fused - left.fused)\n .slice(0, input.limit)\n .map(item => ({\n ...item.result,\n score: item.fused,\n scores: { vector: item.vector, text: item.text, fused: item.fused }\n }))\n}\n\nconst validateSearchInput = (input: KnowledgeSearchInput) => {\n const query = input.query.trim()\n const ids = scopeIds(input.scope)\n const limit = input.limit ?? 10\n const contextChunks = input.contextChunks ?? 0\n const mode = input.mode ?? 'hybrid'\n const vectorLimit = input.vectorLimit ?? (mode === 'hybrid' ? defaultHybridCandidateLimit(limit) : limit)\n const textLimit = input.textLimit ?? defaultHybridCandidateLimit(limit)\n\n if (query.length === 0) {\n return Effect.fail(new KnowledgeSearchError({ message: 'Search query is empty', stage: 'embed' }))\n }\n\n if (ids.length === 0) {\n return Effect.fail(new KnowledgeSearchError({ message: 'Search scope is empty', stage: 'store' }))\n }\n\n if (!Number.isInteger(limit) || limit < 1) {\n return Effect.fail(new KnowledgeSearchError({ message: 'Search limit must be positive', stage: 'store' }))\n }\n\n if (!Number.isInteger(vectorLimit) || vectorLimit < 1) {\n return Effect.fail(new KnowledgeSearchError({ message: 'Vector search limit must be positive', stage: 'store' }))\n }\n\n if (!Number.isInteger(textLimit) || textLimit < 1) {\n return Effect.fail(new KnowledgeSearchError({ message: 'Text search limit must be positive', stage: 'store' }))\n }\n\n if (mode !== 'vector' && mode !== 'hybrid') {\n return Effect.fail(new KnowledgeSearchError({ message: 'Search mode is invalid', stage: 'store' }))\n }\n\n if (!Number.isInteger(contextChunks) || contextChunks < 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Context chunks must be zero or positive', stage: 'store' })\n )\n }\n\n if (input.minScore !== undefined && !Number.isFinite(input.minScore)) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search minScore must be finite', stage: 'store' })\n )\n }\n\n return Effect.succeed({ query, limit, contextChunks, mode, vectorLimit, textLimit })\n}\n\nconst searchVectorChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n readonly minScore?: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n const embedder = yield* KnowledgeEmbedder\n const embedding = yield* embedder\n .embedQuery(input.query)\n .pipe(\n Effect.mapError(\n error => new KnowledgeSearchError({ message: error.message, stage: 'embed', cause: error })\n )\n )\n\n return yield* store\n .searchChunks({\n scope: input.scope,\n embedding,\n limit: input.limit,\n minScore: input.minScore\n })\n .pipe(\n Effect.mapError(\n error => new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nconst searchTextChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n return yield* store\n .searchChunksByText({ scope: input.scope, query: input.query, limit: input.limit })\n .pipe(\n Effect.mapError(\n error => new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nexport const searchKnowledge = (input: KnowledgeSearchInput) =>\n Effect.gen(function* () {\n const valid = yield* validateSearchInput(input)\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.query_length': valid.query.length,\n 'knowledge_search.limit': valid.limit,\n 'knowledge_search.mode': valid.mode,\n 'knowledge_search.context_chunks': valid.contextChunks,\n 'knowledge_search.scope_count': scopeIds(input.scope).length\n })\n const store = yield* SearchIndexStore\n const results: ReadonlyArray<KnowledgeSearchResult> = valid.mode === 'vector'\n ? yield* searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n })\n : yield* Effect.gen(function* () {\n const searches = yield* Effect.all(\n {\n vectorResults: searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n }),\n textResults: searchTextChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.textLimit\n })\n },\n { concurrency: 'unbounded' }\n )\n\n return fuseKnowledgeSearchResults({\n vectorResults: searches.vectorResults,\n textResults: searches.textResults,\n limit: valid.limit\n })\n })\n\n if (valid.contextChunks === 0) {\n return results\n }\n\n return yield* Effect.forEach(results, result =>\n store\n .getContextChunks({\n collectionId: result.chunk.collectionId,\n documentId: result.chunk.documentId,\n position: result.chunk.position,\n contextChunks: valid.contextChunks\n })\n .pipe(\n Effect.map(context => ({ ...result, context })),\n Effect.mapError(\n error => new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n )\n }).pipe(Effect.withSpan('knowledge_search.search'))\n"],"mappings":";;;;;AA6CA,MAAM,oBAAoB,WACxB,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,EAAE,KAAK,MAAM,KAAK,OAAO,MAAM;AAE3E,MAAa,8BAA8B,OAAe,aACvD;CACC;CACA;CACA,MAAM,QAAQ,IAAI,gBAAgB,EAAE,KAAK,MAAM;AACjD;AAEF,MAAM,YAAY,UAAuD;CACvE,QAAQ,MAAM,MAAd;EACE,KAAK,uBACH,OAAO,CAAC,MAAM,EAAE;EAClB,KAAK,wBACH,OAAO,MAAM;CACjB;AACF;AAEA,MAAM,+BAA+B,UAAkB,KAAK,IAAI,QAAQ,GAAG,EAAE;AAE7E,MAAM,6BAA6B,MAAc,iBAAyB,KAAK,eAAe;AAE9F,MAAa,8BAA8B,UAKC;CAC1C,MAAM,eAAe,MAAM,gBAAgB;CAC3C,MAAM,wBAAQ,IAAI,IAQhB;CAEF,MAAM,cAAc,SAA+C,SAA4B;EAC7F,QAAQ,SAAS,QAAQ,UAAU;GACjC,MAAM,OAAO,QAAQ;GACrB,MAAM,WAAW,MAAM,IAAI,OAAO,MAAM,EAAE;GAC1C,MAAM,aAAa,UAAU,SAAS,KAAK,0BAA0B,MAAM,YAAY;GACvF,MAAM,IAAI,OAAO,MAAM,IAAI;IACzB,QAAQ,UAAU,UAAU;IAC5B,QAAQ,SAAS,WAAW,OAAO,QAAQ,UAAU;IACrD,MAAM,SAAS,SAAS,OAAO,QAAQ,UAAU;IACjD,OAAO;GACT,CAAC;EACH,CAAC;CACH;CAEA,WAAW,MAAM,eAAe,QAAQ;CACxC,WAAW,MAAM,aAAa,MAAM;CAEpC,OAAO,MAAM,KAAK,MAAM,OAAO,CAAC,EAC7B,MAAM,MAAM,UAAU,MAAM,QAAQ,KAAK,KAAK,EAC9C,MAAM,GAAG,MAAM,KAAK,EACpB,KAAI,UAAS;EACZ,GAAG,KAAK;EACR,OAAO,KAAK;EACZ,QAAQ;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM,OAAO,KAAK;EAAM;CACpE,EAAE;AACN;AAEA,MAAM,uBAAuB,UAAgC;CAC3D,MAAM,QAAQ,MAAM,MAAM,KAAK;CAC/B,MAAM,MAAM,SAAS,MAAM,KAAK;CAChC,MAAM,QAAQ,MAAM,SAAS;CAC7B,MAAM,gBAAgB,MAAM,iBAAiB;CAC7C,MAAM,OAAO,MAAM,QAAQ;CAC3B,MAAM,cAAc,MAAM,gBAAgB,SAAS,WAAW,4BAA4B,KAAK,IAAI;CACnG,MAAM,YAAY,MAAM,aAAa,4BAA4B,KAAK;CAEtE,IAAI,MAAM,WAAW,GACnB,OAAO,OAAO,KAAK,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAAC;CAGnG,IAAI,IAAI,WAAW,GACjB,OAAO,OAAO,KAAK,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAAC;CAGnG,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GACtC,OAAO,OAAO,KAAK,IAAI,qBAAqB;EAAE,SAAS;EAAiC,OAAO;CAAQ,CAAC,CAAC;CAG3G,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,GAClD,OAAO,OAAO,KAAK,IAAI,qBAAqB;EAAE,SAAS;EAAwC,OAAO;CAAQ,CAAC,CAAC;CAGlH,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,KAAK,IAAI,qBAAqB;EAAE,SAAS;EAAsC,OAAO;CAAQ,CAAC,CAAC;CAGhH,IAAI,SAAS,YAAY,SAAS,UAChC,OAAO,OAAO,KAAK,IAAI,qBAAqB;EAAE,SAAS;EAA0B,OAAO;CAAQ,CAAC,CAAC;CAGpG,IAAI,CAAC,OAAO,UAAU,aAAa,KAAK,gBAAgB,GACtD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAA2C,OAAO;CAAQ,CAAC,CACjG;CAGF,IAAI,MAAM,aAAa,KAAA,KAAa,CAAC,OAAO,SAAS,MAAM,QAAQ,GACjE,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAkC,OAAO;CAAQ,CAAC,CACxF;CAGF,OAAO,OAAO,QAAQ;EAAE;EAAO;EAAO;EAAe;EAAM;EAAa;CAAU,CAAC;AACrF;AAEA,MAAM,sBAAsB,UAM1B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CAErB,MAAM,YAAY,QAAO,OADD,mBAErB,WAAW,MAAM,KAAK,EACtB,KACC,OAAO,UACL,UAAS,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CAC5F,CACF;CAEF,OAAO,OAAO,MACX,aAAa;EACZ,OAAO,MAAM;EACb;EACA,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,EACA,KACC,OAAO,UACL,UAAS,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CAC5F,CACF;AACJ,CAAC;AAEH,MAAM,oBAAoB,UAKxB,OAAO,IAAI,aAAa;CAEtB,OAAO,QAAO,OADO,kBAElB,mBAAmB;EAAE,OAAO,MAAM;EAAO,OAAO,MAAM;EAAO,OAAO,MAAM;CAAM,CAAC,EACjF,KACC,OAAO,UACL,UAAS,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CAC5F,CACF;AACJ,CAAC;AAEH,MAAa,mBAAmB,UAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,oBAAoB,KAAK;CAC9C,OAAO,OAAO,oBAAoB;EAChC,iCAAiC,MAAM,MAAM;EAC7C,0BAA0B,MAAM;EAChC,yBAAyB,MAAM;EAC/B,mCAAmC,MAAM;EACzC,gCAAgC,SAAS,MAAM,KAAK,EAAE;CACxD,CAAC;CACD,MAAM,QAAQ,OAAO;CACrB,MAAM,UAAgD,MAAM,SAAS,WACjE,OAAO,mBAAmB;EAC1B,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,IACC,OAAO,OAAO,IAAI,aAAa;EAC/B,MAAM,WAAW,OAAO,OAAO,IAC7B;GACE,eAAe,mBAAmB;IAChC,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,UAAU,MAAM;GAClB,CAAC;GACD,aAAa,iBAAiB;IAC5B,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;GACf,CAAC;EACH,GACA,EAAE,aAAa,YAAY,CAC7B;EAEA,OAAO,2BAA2B;GAChC,eAAe,SAAS;GACxB,aAAa,SAAS;GACtB,OAAO,MAAM;EACf,CAAC;CACH,CAAC;CAEH,IAAI,MAAM,kBAAkB,GAC1B,OAAO;CAGT,OAAO,OAAO,OAAO,QAAQ,UAAS,WACpC,MACG,iBAAiB;EAChB,cAAc,OAAO,MAAM;EAC3B,YAAY,OAAO,MAAM;EACzB,UAAU,OAAO,MAAM;EACvB,eAAe,MAAM;CACvB,CAAC,EACA,KACC,OAAO,KAAI,aAAY;EAAE,GAAG;EAAQ;CAAQ,EAAE,GAC9C,OAAO,UACL,UAAS,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CAC5F,CACF,CACJ;AACF,CAAC,EAAE,KAAK,OAAO,SAAS,yBAAyB,CAAC"}
|
|
1
|
+
{"version":3,"file":"search.mjs","names":[],"sources":["../src/search.ts"],"sourcesContent":["import { Effect, Match } from 'effect'\nimport type { IndexedKnowledgeDocument, KnowledgeChunk, KnowledgeSearchScope } from './documents.ts'\nimport { KnowledgeEmbedder } from './embeddings.ts'\nimport { KnowledgeSearchError } from './errors.ts'\nimport { SearchIndexStore } from './store.ts'\n\nexport type KnowledgeSearchInput = {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit?: number\n readonly minScore?: number\n readonly contextChunks?: number\n readonly mode?: KnowledgeSearchMode\n readonly vectorLimit?: number\n readonly textLimit?: number\n}\n\nexport type KnowledgeSearchMode = 'vector' | 'hybrid'\n\nexport type KnowledgeSearchScores = {\n readonly vector?: number\n readonly text?: number\n readonly fused?: number\n}\n\nexport type KnowledgeSearchResult = {\n readonly chunk: KnowledgeChunk\n readonly score: number\n readonly document: IndexedKnowledgeDocument\n readonly context?: ReadonlyArray<KnowledgeChunk>\n readonly scores?: KnowledgeSearchScores\n}\n\nexport type KnowledgeSearchContext = {\n readonly query: string\n readonly results: ReadonlyArray<KnowledgeSearchResult>\n readonly text: string\n}\n\nexport type KnowledgeSearcher = {\n readonly search: (\n input: KnowledgeSearchInput\n ) => Effect.Effect<ReadonlyArray<KnowledgeSearchResult>, KnowledgeSearchError>\n}\n\nconst packedResultText = (result: KnowledgeSearchResult) =>\n result.context?.map(chunk => chunk.content).join('\\n\\n') ?? result.chunk.content\n\nexport const packKnowledgeSearchContext = (\n query: string,\n results: ReadonlyArray<KnowledgeSearchResult>\n): KnowledgeSearchContext => ({\n query,\n results,\n text: results.map(packedResultText).join('\\n\\n')\n})\n\nconst scopeIds = (scope: KnowledgeSearchScope): ReadonlyArray<string> =>\n Match.value(scope).pipe(\n Match.tagsExhaustive({\n KnowledgeScope: ({ id }) => [id],\n KnowledgeScopes: ({ ids }) => ids\n })\n )\n\nconst defaultHybridCandidateLimit = (limit: number) => Math.max(limit * 5, 40)\n\nconst reciprocalRankFusionScore = (rank: number, rankConstant: number) => 1 / (rankConstant + rank)\n\nexport const fuseKnowledgeSearchResults = (input: {\n readonly vectorResults: ReadonlyArray<KnowledgeSearchResult>\n readonly textResults: ReadonlyArray<KnowledgeSearchResult>\n readonly limit: number\n readonly rankConstant?: number\n}): ReadonlyArray<KnowledgeSearchResult> => {\n const rankConstant = input.rankConstant ?? 60\n\n const fused = new Map<\n string,\n {\n readonly result: KnowledgeSearchResult\n readonly vector?: number\n readonly text?: number\n readonly fused: number\n }\n >()\n\n const addResults = (results: ReadonlyArray<KnowledgeSearchResult>, kind: 'vector' | 'text') => {\n results.forEach((result, index) => {\n const rank = index + 1\n const previous = fused.get(result.chunk.id)\n const nextFused = (previous?.fused ?? 0) + reciprocalRankFusionScore(rank, rankConstant)\n fused.set(result.chunk.id, {\n result: previous?.result ?? result,\n vector: kind === 'vector' ? result.score : previous?.vector,\n text: kind === 'text' ? result.score : previous?.text,\n fused: nextFused\n })\n })\n }\n\n addResults(input.vectorResults, 'vector')\n addResults(input.textResults, 'text')\n\n return Array.from(fused.values())\n .sort((left, right) => right.fused - left.fused)\n .slice(0, input.limit)\n .map(item => ({\n ...item.result,\n score: item.fused,\n scores: { vector: item.vector, text: item.text, fused: item.fused }\n }))\n}\n\nconst validateSearchInput = (input: KnowledgeSearchInput) => {\n const query = input.query.trim()\n const ids = scopeIds(input.scope)\n const limit = input.limit ?? 10\n const contextChunks = input.contextChunks ?? 0\n const mode = input.mode ?? 'hybrid'\n\n const vectorLimit =\n input.vectorLimit ?? (mode === 'hybrid' ? defaultHybridCandidateLimit(limit) : limit)\n\n const textLimit = input.textLimit ?? defaultHybridCandidateLimit(limit)\n\n if (query.length === 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search query is empty', stage: 'embed' })\n )\n }\n\n if (ids.length === 0) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search scope is empty', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(limit) || limit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search limit must be positive', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(vectorLimit) || vectorLimit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Vector search limit must be positive', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(textLimit) || textLimit < 1) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Text search limit must be positive', stage: 'store' })\n )\n }\n\n if (mode !== 'vector' && mode !== 'hybrid') {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search mode is invalid', stage: 'store' })\n )\n }\n\n if (!Number.isInteger(contextChunks) || contextChunks < 0) {\n return Effect.fail(\n new KnowledgeSearchError({\n message: 'Context chunks must be zero or positive',\n stage: 'store'\n })\n )\n }\n\n if (input.minScore !== undefined && !Number.isFinite(input.minScore)) {\n return Effect.fail(\n new KnowledgeSearchError({ message: 'Search minScore must be finite', stage: 'store' })\n )\n }\n\n return Effect.succeed({ query, limit, contextChunks, mode, vectorLimit, textLimit })\n}\n\nconst searchVectorChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n readonly minScore?: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n const embedder = yield* KnowledgeEmbedder\n\n const embedding = yield* embedder\n .embedQuery(input.query)\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'embed', cause: error })\n )\n )\n\n return yield* store\n .searchChunks({\n scope: input.scope,\n embedding,\n limit: input.limit,\n minScore: input.minScore\n })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nconst searchTextChunks = (input: {\n readonly scope: KnowledgeSearchScope\n readonly query: string\n readonly limit: number\n}) =>\n Effect.gen(function* () {\n const store = yield* SearchIndexStore\n\n return yield* store\n .searchChunksByText({ scope: input.scope, query: input.query, limit: input.limit })\n .pipe(\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n })\n\nexport const searchKnowledge = (input: KnowledgeSearchInput) =>\n Effect.gen(function* () {\n const valid = yield* validateSearchInput(input)\n yield* Effect.annotateCurrentSpan({\n 'knowledge_search.query_length': valid.query.length,\n 'knowledge_search.limit': valid.limit,\n 'knowledge_search.mode': valid.mode,\n 'knowledge_search.context_chunks': valid.contextChunks,\n 'knowledge_search.scope_count': scopeIds(input.scope).length\n })\n const store = yield* SearchIndexStore\n\n const results: ReadonlyArray<KnowledgeSearchResult> =\n valid.mode === 'vector'\n ? yield* searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n })\n : yield* Effect.gen(function* () {\n const searches = yield* Effect.all(\n {\n vectorResults: searchVectorChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.vectorLimit,\n minScore: input.minScore\n }),\n textResults: searchTextChunks({\n scope: input.scope,\n query: valid.query,\n limit: valid.textLimit\n })\n },\n { concurrency: 'unbounded' }\n )\n\n return fuseKnowledgeSearchResults({\n vectorResults: searches.vectorResults,\n textResults: searches.textResults,\n limit: valid.limit\n })\n })\n\n if (valid.contextChunks === 0) {\n return results\n }\n\n return yield* Effect.forEach(results, result =>\n store\n .getContextChunks({\n scopeId: result.chunk.scopeId,\n documentId: result.chunk.documentId,\n position: result.chunk.position,\n contextChunks: valid.contextChunks\n })\n .pipe(\n Effect.map(context => ({ ...result, context })),\n Effect.mapError(\n error =>\n new KnowledgeSearchError({ message: error.message, stage: 'store', cause: error })\n )\n )\n )\n }).pipe(Effect.withSpan('knowledge_search.search'))\n"],"mappings":";;;;;AA6CA,MAAM,oBAAoB,WACxB,OAAO,SAAS,KAAI,UAAS,MAAM,OAAO,CAAC,CAAC,KAAK,MAAM,KAAK,OAAO,MAAM;AAE3E,MAAa,8BACX,OACA,aAC4B;CAC5B;CACA;CACA,MAAM,QAAQ,IAAI,gBAAgB,CAAC,CAAC,KAAK,MAAM;AACjD;AAEA,MAAM,YAAY,UAChB,MAAM,MAAM,KAAK,CAAC,CAAC,KACjB,MAAM,eAAe;CACnB,iBAAiB,EAAE,SAAS,CAAC,EAAE;CAC/B,kBAAkB,EAAE,UAAU;AAChC,CAAC,CACH;AAEF,MAAM,+BAA+B,UAAkB,KAAK,IAAI,QAAQ,GAAG,EAAE;AAE7E,MAAM,6BAA6B,MAAc,iBAAyB,KAAK,eAAe;AAE9F,MAAa,8BAA8B,UAKC;CAC1C,MAAM,eAAe,MAAM,gBAAgB;CAE3C,MAAM,wBAAQ,IAAI,IAQhB;CAEF,MAAM,cAAc,SAA+C,SAA4B;EAC7F,QAAQ,SAAS,QAAQ,UAAU;GACjC,MAAM,OAAO,QAAQ;GACrB,MAAM,WAAW,MAAM,IAAI,OAAO,MAAM,EAAE;GAC1C,MAAM,aAAa,UAAU,SAAS,KAAK,0BAA0B,MAAM,YAAY;GACvF,MAAM,IAAI,OAAO,MAAM,IAAI;IACzB,QAAQ,UAAU,UAAU;IAC5B,QAAQ,SAAS,WAAW,OAAO,QAAQ,UAAU;IACrD,MAAM,SAAS,SAAS,OAAO,QAAQ,UAAU;IACjD,OAAO;GACT,CAAC;EACH,CAAC;CACH;CAEA,WAAW,MAAM,eAAe,QAAQ;CACxC,WAAW,MAAM,aAAa,MAAM;CAEpC,OAAO,MAAM,KAAK,MAAM,OAAO,CAAC,CAAC,CAC9B,MAAM,MAAM,UAAU,MAAM,QAAQ,KAAK,KAAK,CAAC,CAC/C,MAAM,GAAG,MAAM,KAAK,CAAC,CACrB,KAAI,UAAS;EACZ,GAAG,KAAK;EACR,OAAO,KAAK;EACZ,QAAQ;GAAE,QAAQ,KAAK;GAAQ,MAAM,KAAK;GAAM,OAAO,KAAK;EAAM;CACpE,EAAE;AACN;AAEA,MAAM,uBAAuB,UAAgC;CAC3D,MAAM,QAAQ,MAAM,MAAM,KAAK;CAC/B,MAAM,MAAM,SAAS,MAAM,KAAK;CAChC,MAAM,QAAQ,MAAM,SAAS;CAC7B,MAAM,gBAAgB,MAAM,iBAAiB;CAC7C,MAAM,OAAO,MAAM,QAAQ;CAE3B,MAAM,cACJ,MAAM,gBAAgB,SAAS,WAAW,4BAA4B,KAAK,IAAI;CAEjF,MAAM,YAAY,MAAM,aAAa,4BAA4B,KAAK;CAEtE,IAAI,MAAM,WAAW,GACnB,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAC/E;CAGF,IAAI,IAAI,WAAW,GACjB,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAyB,OAAO;CAAQ,CAAC,CAC/E;CAGF,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GACtC,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAiC,OAAO;CAAQ,CAAC,CACvF;CAGF,IAAI,CAAC,OAAO,UAAU,WAAW,KAAK,cAAc,GAClD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAwC,OAAO;CAAQ,CAAC,CAC9F;CAGF,IAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,GAC9C,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAsC,OAAO;CAAQ,CAAC,CAC5F;CAGF,IAAI,SAAS,YAAY,SAAS,UAChC,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAA0B,OAAO;CAAQ,CAAC,CAChF;CAGF,IAAI,CAAC,OAAO,UAAU,aAAa,KAAK,gBAAgB,GACtD,OAAO,OAAO,KACZ,IAAI,qBAAqB;EACvB,SAAS;EACT,OAAO;CACT,CAAC,CACH;CAGF,IAAI,MAAM,aAAa,KAAA,KAAa,CAAC,OAAO,SAAS,MAAM,QAAQ,GACjE,OAAO,OAAO,KACZ,IAAI,qBAAqB;EAAE,SAAS;EAAkC,OAAO;CAAQ,CAAC,CACxF;CAGF,OAAO,OAAO,QAAQ;EAAE;EAAO;EAAO;EAAe;EAAM;EAAa;CAAU,CAAC;AACrF;AAEA,MAAM,sBAAsB,UAM1B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO;CAGrB,MAAM,YAAY,QAAO,OAFD,kBAAA,CAGrB,WAAW,MAAM,KAAK,CAAC,CACvB,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;CAEF,OAAO,OAAO,MACX,aAAa;EACZ,OAAO,MAAM;EACb;EACA,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,CAAC,CACD,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;AACJ,CAAC;AAEH,MAAM,oBAAoB,UAKxB,OAAO,IAAI,aAAa;CAGtB,OAAO,QAAO,OAFO,iBAAA,CAGlB,mBAAmB;EAAE,OAAO,MAAM;EAAO,OAAO,MAAM;EAAO,OAAO,MAAM;CAAM,CAAC,CAAC,CAClF,KACC,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF;AACJ,CAAC;AAEH,MAAa,mBAAmB,UAC9B,OAAO,IAAI,aAAa;CACtB,MAAM,QAAQ,OAAO,oBAAoB,KAAK;CAC9C,OAAO,OAAO,oBAAoB;EAChC,iCAAiC,MAAM,MAAM;EAC7C,0BAA0B,MAAM;EAChC,yBAAyB,MAAM;EAC/B,mCAAmC,MAAM;EACzC,gCAAgC,SAAS,MAAM,KAAK,CAAC,CAAC;CACxD,CAAC;CACD,MAAM,QAAQ,OAAO;CAErB,MAAM,UACJ,MAAM,SAAS,WACX,OAAO,mBAAmB;EACxB,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM;EACb,UAAU,MAAM;CAClB,CAAC,IACD,OAAO,OAAO,IAAI,aAAa;EAC7B,MAAM,WAAW,OAAO,OAAO,IAC7B;GACE,eAAe,mBAAmB;IAChC,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;IACb,UAAU,MAAM;GAClB,CAAC;GACD,aAAa,iBAAiB;IAC5B,OAAO,MAAM;IACb,OAAO,MAAM;IACb,OAAO,MAAM;GACf,CAAC;EACH,GACA,EAAE,aAAa,YAAY,CAC7B;EAEA,OAAO,2BAA2B;GAChC,eAAe,SAAS;GACxB,aAAa,SAAS;GACtB,OAAO,MAAM;EACf,CAAC;CACH,CAAC;CAEP,IAAI,MAAM,kBAAkB,GAC1B,OAAO;CAGT,OAAO,OAAO,OAAO,QAAQ,UAAS,WACpC,MACG,iBAAiB;EAChB,SAAS,OAAO,MAAM;EACtB,YAAY,OAAO,MAAM;EACzB,UAAU,OAAO,MAAM;EACvB,eAAe,MAAM;CACvB,CAAC,CAAC,CACD,KACC,OAAO,KAAI,aAAY;EAAE,GAAG;EAAQ;CAAQ,EAAE,GAC9C,OAAO,UACL,UACE,IAAI,qBAAqB;EAAE,SAAS,MAAM;EAAS,OAAO;EAAS,OAAO;CAAM,CAAC,CACrF,CACF,CACJ;AACF,CAAC,CAAC,CAAC,KAAK,OAAO,SAAS,yBAAyB,CAAC"}
|