@shenghuabi/knowledge 1.0.21

package/knowledge/common/common.knowledge.service.d.ts

@@ -0,0 +1,240 @@
+import { FilterOptions, QueryOptions } from './query';
+export declare class CommonKnowledgeService {
+    #private;
+    searchChunk(text: string, filter?: FilterOptions, options?: QueryOptions): Promise<{
+        id: string | number;
+        version: number;
+        score: number;
+        payload?: Record<string, unknown> | {
+            [key: string]: unknown;
+        } | null | undefined;
+        vector?: Record<string, unknown> | number[] | number[][] | {
+            [key: string]: number[] | number[][] | {
+                indices: number[];
+                values: number[];
+            } | undefined;
+        } | null | undefined;
+        shard_key?: string | number | Record<string, unknown> | null | undefined;
+        order_value?: number | Record<string, unknown> | null | undefined;
+    }[]>;
+    getCollection(): Promise<{
+        status: "green" | "yellow" | "grey" | "red";
+        optimizer_status: "ok" | {
+            error: string;
+        };
+        vectors_count?: number | null | undefined;
+        indexed_vectors_count?: number | null | undefined;
+        points_count?: number | null | undefined;
+        segments_count: number;
+        config: {
+            params: {
+                vectors?: {
+                    size: number;
+                    distance: "Cosine" | "Euclid" | "Dot" | "Manhattan";
+                    hnsw_config?: Record<string, unknown> | {
+                        m?: number | null | undefined;
+                        ef_construct?: number | null | undefined;
+                        full_scan_threshold?: number | null | undefined;
+                        max_indexing_threads?: number | null | undefined;
+                        on_disk?: boolean | null | undefined;
+                        payload_m?: number | null | undefined;
+                    } | null | undefined;
+                    quantization_config?: Record<string, unknown> | {
+                        scalar: {
+                            type: "int8";
+                            quantile?: number | null | undefined;
+                            always_ram?: boolean | null | undefined;
+                        };
+                    } | {
+                        product: {
+                            compression: "x4" | "x8" | "x16" | "x32" | "x64";
+                            always_ram?: boolean | null | undefined;
+                        };
+                    } | {
+                        binary: {
+                            always_ram?: boolean | null | undefined;
+                            encoding?: Record<string, unknown> | "one_bit" | "two_bits" | "one_and_half_bits" | null | undefined;
+                            query_encoding?: Record<string, unknown> | "default" | "binary" | "scalar4bits" | "scalar8bits" | null | undefined;
+                        };
+                    } | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                    datatype?: Record<string, unknown> | "float32" | "uint8" | "float16" | null | undefined;
+                    multivector_config?: Record<string, unknown> | {
+                        comparator: "max_sim";
+                    } | null | undefined;
+                } | {
+                    [key: string]: {
+                        size: number;
+                        distance: "Cosine" | "Euclid" | "Dot" | "Manhattan";
+                        hnsw_config?: Record<string, unknown> | {
+                            m?: number | null | undefined;
+                            ef_construct?: number | null | undefined;
+                            full_scan_threshold?: number | null | undefined;
+                            max_indexing_threads?: number | null | undefined;
+                            on_disk?: boolean | null | undefined;
+                            payload_m?: number | null | undefined;
+                        } | null | undefined;
+                        quantization_config?: Record<string, unknown> | {
+                            scalar: {
+                                type: "int8";
+                                quantile?: number | null | undefined;
+                                always_ram?: boolean | null | undefined;
+                            };
+                        } | {
+                            product: {
+                                compression: "x4" | "x8" | "x16" | "x32" | "x64";
+                                always_ram?: boolean | null | undefined;
+                            };
+                        } | {
+                            binary: {
+                                always_ram?: boolean | null | undefined;
+                                encoding?: Record<string, unknown> | "one_bit" | "two_bits" | "one_and_half_bits" | null | undefined;
+                                query_encoding?: Record<string, unknown> | "default" | "binary" | "scalar4bits" | "scalar8bits" | null | undefined;
+                            };
+                        } | null | undefined;
+                        on_disk?: boolean | null | undefined;
+                        datatype?: Record<string, unknown> | "float32" | "uint8" | "float16" | null | undefined;
+                        multivector_config?: Record<string, unknown> | {
+                            comparator: "max_sim";
+                        } | null | undefined;
+                    } | undefined;
+                } | undefined;
+                shard_number?: number | undefined;
+                sharding_method?: Record<string, unknown> | "auto" | "custom" | null | undefined;
+                replication_factor?: number | undefined;
+                write_consistency_factor?: number | undefined;
+                read_fan_out_factor?: number | null | undefined;
+                on_disk_payload?: boolean | undefined;
+                sparse_vectors?: {
+                    [key: string]: {
+                        index?: Record<string, unknown> | {
+                            full_scan_threshold?: number | null | undefined;
+                            on_disk?: boolean | null | undefined;
+                            datatype?: Record<string, unknown> | "float32" | "uint8" | "float16" | null | undefined;
+                        } | null | undefined;
+                        modifier?: Record<string, unknown> | "none" | "idf" | null | undefined;
+                    } | undefined;
+                } | null | undefined;
+            };
+            hnsw_config: {
+                m: number;
+                ef_construct: number;
+                full_scan_threshold: number;
+                max_indexing_threads?: number | undefined;
+                on_disk?: boolean | null | undefined;
+                payload_m?: number | null | undefined;
+            };
+            optimizer_config: {
+                deleted_threshold: number;
+                vacuum_min_vector_number: number;
+                default_segment_number: number;
+                max_segment_size?: number | null | undefined;
+                memmap_threshold?: number | null | undefined;
+                indexing_threshold?: number | null | undefined;
+                flush_interval_sec: number;
+                max_optimization_threads?: number | null | undefined;
+            };
+            wal_config?: Record<string, unknown> | {
+                wal_capacity_mb: number;
+                wal_segments_ahead: number;
+            } | null | undefined;
+            quantization_config?: Record<string, unknown> | {
+                scalar: {
+                    type: "int8";
+                    quantile?: number | null | undefined;
+                    always_ram?: boolean | null | undefined;
+                };
+            } | {
+                product: {
+                    compression: "x4" | "x8" | "x16" | "x32" | "x64";
+                    always_ram?: boolean | null | undefined;
+                };
+            } | {
+                binary: {
+                    always_ram?: boolean | null | undefined;
+                    encoding?: Record<string, unknown> | "one_bit" | "two_bits" | "one_and_half_bits" | null | undefined;
+                    query_encoding?: Record<string, unknown> | "default" | "binary" | "scalar4bits" | "scalar8bits" | null | undefined;
+                };
+            } | null | undefined;
+            strict_mode_config?: Record<string, unknown> | {
+                enabled?: boolean | null | undefined;
+                max_query_limit?: number | null | undefined;
+                max_timeout?: number | null | undefined;
+                unindexed_filtering_retrieve?: boolean | null | undefined;
+                unindexed_filtering_update?: boolean | null | undefined;
+                search_max_hnsw_ef?: number | null | undefined;
+                search_allow_exact?: boolean | null | undefined;
+                search_max_oversampling?: number | null | undefined;
+                upsert_max_batchsize?: number | null | undefined;
+                max_collection_vector_size_bytes?: number | null | undefined;
+                read_rate_limit?: number | null | undefined;
+                write_rate_limit?: number | null | undefined;
+                max_collection_payload_size_bytes?: number | null | undefined;
+                max_points_count?: number | null | undefined;
+                filter_max_conditions?: number | null | undefined;
+                condition_max_size?: number | null | undefined;
+                multivector_config?: Record<string, unknown> | {
+                    [key: string]: {
+                        max_vectors?: number | null | undefined;
+                    } | undefined;
+                } | null | undefined;
+                sparse_config?: Record<string, unknown> | {
+                    [key: string]: {
+                        max_length?: number | null | undefined;
+                    } | undefined;
+                } | null | undefined;
+            } | null | undefined;
+        };
+        payload_schema: {
+            [key: string]: {
+                data_type: "keyword" | "integer" | "float" | "geo" | "text" | "bool" | "datetime" | "uuid";
+                params?: Record<string, unknown> | {
+                    type: "keyword";
+                    is_tenant?: boolean | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                } | {
+                    type: "integer";
+                    lookup?: boolean | null | undefined;
+                    range?: boolean | null | undefined;
+                    is_principal?: boolean | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                } | {
+                    type: "float";
+                    is_principal?: boolean | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                } | {
+                    type: "geo";
+                    on_disk?: boolean | null | undefined;
+                } | {
+                    type: "text";
+                    tokenizer?: "prefix" | "whitespace" | "word" | "multilingual" | undefined;
+                    min_token_len?: number | null | undefined;
+                    max_token_len?: number | null | undefined;
+                    lowercase?: boolean | null | undefined;
+                    phrase_matching?: boolean | null | undefined;
+                    stopwords?: Record<string, unknown> | "arabic" | "azerbaijani" | "basque" | "bengali" | "catalan" | "chinese" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hebrew" | "hinglish" | "hungarian" | "indonesian" | "italian" | "japanese" | "kazakh" | "nepali" | "norwegian" | "portuguese" | "romanian" | "russian" | "slovene" | "spanish" | "swedish" | "tajik" | "turkish" | {
+                        languages?: ("arabic" | "azerbaijani" | "basque" | "bengali" | "catalan" | "chinese" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hebrew" | "hinglish" | "hungarian" | "indonesian" | "italian" | "japanese" | "kazakh" | "nepali" | "norwegian" | "portuguese" | "romanian" | "russian" | "slovene" | "spanish" | "swedish" | "tajik" | "turkish")[] | null | undefined;
+                        custom?: string[] | null | undefined;
+                    } | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                    stemmer?: Record<string, unknown> | {
+                        type: "snowball";
+                        language: "arabic" | "danish" | "dutch" | "english" | "finnish" | "french" | "german" | "greek" | "hungarian" | "italian" | "norwegian" | "portuguese" | "romanian" | "russian" | "spanish" | "swedish" | "turkish" | "armenian" | "tamil";
+                    } | null | undefined;
+                } | {
+                    type: "bool";
+                    on_disk?: boolean | null | undefined;
+                } | {
+                    type: "datetime";
+                    is_principal?: boolean | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                } | {
+                    type: "uuid";
+                    is_tenant?: boolean | null | undefined;
+                    on_disk?: boolean | null | undefined;
+                } | null | undefined;
+                points: number;
+            } | undefined;
+        };
+    }>;
+}

package/knowledge/common/define/base.d.ts

@@ -0,0 +1,7 @@
+import * as v from 'valibot';
+export declare const BaseKnowledgeConfig: v.ObjectSchema<{
+    /** 知识库名 */
+    readonly name: v.StringSchema<undefined>;
+    readonly activateCollection: v.StringSchema<undefined>;
+}, undefined>;
+export type BaseKnowledgeConfigType = v.InferOutput<typeof BaseKnowledgeConfig>;

package/knowledge/common/define/chunk.d.ts

@@ -0,0 +1,14 @@
+import * as v from 'valibot';
+export declare const FileChunkPayloadDefine: v.ObjectSchema<{
+    readonly chunk: v.StringSchema<undefined>;
+    readonly fileName: v.StringSchema<undefined>;
+    readonly loc: v.CustomSchema<{
+        lines: {
+            from: number;
+            to: number;
+        };
+    }, undefined>;
+    readonly hash: v.StringSchema<undefined>;
+    readonly embeddingChunk: v.StringSchema<undefined>;
+}, undefined>;
+export type FileChunkPayload = v.InferOutput<typeof FileChunkPayloadDefine>;

package/knowledge/common/define/embedding.d.ts

@@ -0,0 +1,5 @@
+import * as v from 'valibot';
+export declare const EmbeddingTemplateDefine: v.ObjectSchema<{
+    readonly enable: v.BooleanSchema<undefined>;
+    readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+}, undefined>;

package/knowledge/common/define/index.d.ts

@@ -0,0 +1,3 @@
+export * from './base';
+export * from './chunk';
+export * from './embedding';

package/knowledge/common/index.d.ts

@@ -0,0 +1 @@
+export * from './define';

package/knowledge/common/query.d.ts

@@ -0,0 +1,7 @@
+import { QdrantClient } from '@qdrant/qdrant-js';
+export interface QueryOptions {
+    limit?: number;
+    score?: number;
+    offset?: number;
+}
+export type FilterOptions = Parameters<QdrantClient['search']>[1]['filter'];

package/knowledge/const.d.ts

@@ -0,0 +1,95 @@
+import { InjectionToken, Signal } from 'static-injector';
+import { KnowledgeConfig, OCR, Text2Vec, TextSplitter } from './type';
+export declare function getGraphCollectionName(name: string): string;
+/** 进行增加删除的永远是这个 */
+export declare function getActivateCollectionName(name: string): string;
+export declare const DICT_PREFIX = "[\u5B57\u5178]-";
+export declare const Text2VecToken: InjectionToken<Text2Vec>;
+export declare const TextSplitterToken: InjectionToken<TextSplitter>;
+export declare const ConfigToken: InjectionToken<Signal<{
+    activateName: string;
+    type: "normal";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+} | {
+    activateName: string;
+    type: "dict";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    extractorWord?: boolean | undefined;
+    useOcr?: boolean | undefined;
+    name: string;
+    activateCollection: string;
+} | {
+    activateName: string;
+    activateGraphName: string;
+    type: "normal-graph";
+    maxChunkAsync: number;
+    collectionList: {
+        graphCollectionName: string;
+        collectionName: string;
+        size: number;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            node?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            edge?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+    }[];
+    name: string;
+    activateCollection: string;
+} | {
+    activateName: string;
+    type: "article";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}>>;
+export declare const GetConfigToken: InjectionToken<(name: string) => Promise<KnowledgeConfig>>;
+export declare const OCRToken: InjectionToken<OCR>;
+export declare const DirToken: InjectionToken<Signal<string>>;
+export declare const ReRankerToken: InjectionToken<{
+    run: (text: {
+        value: string;
+        docs: string[];
+    }) => Promise<{
+        index: number;
+        score: any;
+    }[]>;
+    getQueryRatio(): number;
+}>;

package/knowledge/define/index.d.ts

@@ -0,0 +1,245 @@
+import * as v from 'valibot';
+export declare const KnowledgeConfigDefine: v.UnionSchema<[v.SchemaWithPipe<readonly [v.ObjectSchema<{
+    readonly type: v.OptionalSchema<v.LiteralSchema<"normal", undefined>, "normal">;
+    readonly collectionList: v.ArraySchema<v.ObjectSchema<{
+        readonly collectionName: v.StringSchema<undefined>;
+        readonly embeddingTemplate: v.OptionalSchema<v.ObjectSchema<{
+            readonly entry: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+        }, undefined>, undefined>;
+        readonly size: v.NumberSchema<undefined>;
+    }, undefined>, undefined>;
+    readonly name: v.StringSchema<undefined>;
+    readonly activateCollection: v.StringSchema<undefined>;
+}, undefined>, v.TransformAction<{
+    type: "normal";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}, {
+    activateName: string;
+    type: "normal";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}>]>, v.SchemaWithPipe<readonly [v.ObjectSchema<{
+    readonly type: v.OptionalSchema<v.LiteralSchema<"dict", undefined>, "dict">;
+    readonly collectionList: v.ArraySchema<v.ObjectSchema<{
+        readonly collectionName: v.StringSchema<undefined>;
+        readonly embeddingTemplate: v.OptionalSchema<v.ObjectSchema<{
+            readonly entry: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+        }, undefined>, undefined>;
+        readonly size: v.NumberSchema<undefined>;
+    }, undefined>, undefined>;
+    readonly extractorWord: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+    readonly useOcr: v.OptionalSchema<v.BooleanSchema<undefined>, undefined>;
+    readonly name: v.StringSchema<undefined>;
+    readonly activateCollection: v.StringSchema<undefined>;
+}, undefined>, v.TransformAction<{
+    type: "dict";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    extractorWord?: boolean | undefined;
+    useOcr?: boolean | undefined;
+    name: string;
+    activateCollection: string;
+}, {
+    activateName: string;
+    type: "dict";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    extractorWord?: boolean | undefined;
+    useOcr?: boolean | undefined;
+    name: string;
+    activateCollection: string;
+}>]>, v.SchemaWithPipe<readonly [v.ObjectSchema<{
+    readonly type: v.OptionalSchema<v.LiteralSchema<"normal-graph", undefined>, "normal-graph">;
+    readonly maxChunkAsync: v.NumberSchema<undefined>;
+    readonly collectionList: v.ArraySchema<v.SchemaWithPipe<readonly [v.ObjectSchema<{
+        readonly collectionName: v.StringSchema<undefined>;
+        readonly size: v.NumberSchema<undefined>;
+        readonly embeddingTemplate: v.OptionalSchema<v.ObjectSchema<{
+            readonly entry: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+            readonly node: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+            readonly edge: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+        }, undefined>, undefined>;
+    }, undefined>, v.TransformAction<{
+        collectionName: string;
+        size: number;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            node?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            edge?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+    }, {
+        graphCollectionName: string;
+        collectionName: string;
+        size: number;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            node?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            edge?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+    }>]>, undefined>;
+    readonly name: v.StringSchema<undefined>;
+    readonly activateCollection: v.StringSchema<undefined>;
+}, undefined>, v.TransformAction<{
+    type: "normal-graph";
+    maxChunkAsync: number;
+    collectionList: {
+        graphCollectionName: string;
+        collectionName: string;
+        size: number;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            node?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            edge?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+    }[];
+    name: string;
+    activateCollection: string;
+}, {
+    activateName: string;
+    activateGraphName: string;
+    type: "normal-graph";
+    maxChunkAsync: number;
+    collectionList: {
+        graphCollectionName: string;
+        collectionName: string;
+        size: number;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            node?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+            edge?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+    }[];
+    name: string;
+    activateCollection: string;
+}>]>, v.SchemaWithPipe<readonly [v.ObjectSchema<{
+    readonly type: v.OptionalSchema<v.LiteralSchema<"article", undefined>, "article">;
+    readonly collectionList: v.ArraySchema<v.ObjectSchema<{
+        readonly collectionName: v.StringSchema<undefined>;
+        readonly embeddingTemplate: v.OptionalSchema<v.ObjectSchema<{
+            readonly entry: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+        }, undefined>, undefined>;
+        readonly size: v.NumberSchema<undefined>;
+    }, undefined>, undefined>;
+    readonly name: v.StringSchema<undefined>;
+    readonly activateCollection: v.StringSchema<undefined>;
+}, undefined>, v.TransformAction<{
+    type: "article";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}, {
+    activateName: string;
+    type: "article";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}>]>], undefined>;