@shenghuabi/knowledge 1.0.21

package/knowledge/graph/graph.query.service.d.ts

@@ -0,0 +1,160 @@
+import { FileChunkPayload } from '../common/define/chunk';
+export declare class GraphQueryService {
+    #private;
+    query(params: {
+        node?: string;
+        edge?: string;
+    }, question: string): Promise<{
+        nodes: {
+            list: {
+                kind: "node";
+                id: string;
+                chunkId: string;
+                fileName: string;
+                name: string;
+                description: string;
+                type: string;
+            }[];
+            name: string;
+            type: string;
+            degree: number;
+        }[];
+        edges: {
+            list: {
+                chunkId: string;
+                kind: "edge";
+                name: string;
+                fileName: string;
+                id: string;
+                source: string;
+                target: string;
+                description: string;
+                strength: number;
+                keywords?: string[] | undefined;
+            }[];
+            name: string;
+            source: string;
+            target: string;
+            degree: number;
+        }[];
+        chunks: {
+            chunk: string;
+            fileName: string;
+            loc: {
+                lines: {
+                    from: number;
+                    to: number;
+                };
+            };
+            hash: string;
+            embeddingChunk: string;
+        }[];
+    }>;
+    searchNode(str: string, selectedList: string[]): Promise<string[]>;
+    getFileNameList(): Promise<{
+        hits: {
+            id: string | number;
+            version: number;
+            score: number;
+            payload?: Record<string, unknown> | {
+                [key: string]: unknown;
+            } | null | undefined;
+            vector?: Record<string, unknown> | number[] | number[][] | {
+                [key: string]: number[] | number[][] | {
+                    indices: number[];
+                    values: number[];
+                } | undefined;
+            } | null | undefined;
+            shard_key?: string | number | Record<string, unknown> | null | undefined;
+            order_value?: number | Record<string, unknown> | null | undefined;
+        }[];
+        id: string | number;
+        lookup?: Record<string, unknown> | {
+            id: string | number;
+            payload?: Record<string, unknown> | {
+                [key: string]: unknown;
+            } | null | undefined;
+            vector?: Record<string, unknown> | number[] | number[][] | {
+                [key: string]: number[] | number[][] | {
+                    indices: number[];
+                    values: number[];
+                } | undefined;
+            } | null | undefined;
+            shard_key?: string | number | Record<string, unknown> | null | undefined;
+            order_value?: number | Record<string, unknown> | null | undefined;
+        } | null | undefined;
+    }[]>;
+    getChunkContent(fileName: string): Promise<{
+        payload: FileChunkPayload & {
+            knowledge: string;
+        };
+        id: string | number;
+        version: number;
+        score: number;
+        vector?: Record<string, unknown> | number[] | number[][] | {
+            [key: string]: number[] | number[][] | {
+                indices: number[];
+                values: number[];
+            } | undefined;
+        } | null | undefined;
+        shard_key?: string | number | Record<string, unknown> | null | undefined;
+        order_value?: number | Record<string, unknown> | null | undefined;
+    }[]>;
+    getGraphData(): Promise<import("graphology-types").SerializedGraph<import("./define").GraphNodeAttr, import("./define").GraphEdgeAttr, import("graphology-types").Attributes>>;
+    queryNode(list: string[], options: {
+        nodeSizeLimit: number;
+    }): Promise<{
+        id: string | number;
+        payload?: Record<string, unknown> | {
+            [key: string]: unknown;
+        } | null | undefined;
+        vector?: Record<string, unknown> | number[] | number[][] | {
+            [key: string]: number[] | number[][] | {
+                indices: number[];
+                values: number[];
+            } | undefined;
+        } | null | undefined;
+        shard_key?: string | number | Record<string, unknown> | null | undefined;
+        order_value?: number | Record<string, unknown> | null | undefined;
+    }[]>;
+    fuzzyQueryNode(content: string, options: {
+        /** 限制多少个节点 */
+        nodeLimit: number;
+        /** 精度 */
+        score: number;
+        /** 限制每个节点多少信息 */
+        nodeSizeLimit: number;
+    }): Promise<{
+        hits: {
+            id: string | number;
+            version: number;
+            score: number;
+            payload?: Record<string, unknown> | {
+                [key: string]: unknown;
+            } | null | undefined;
+            vector?: Record<string, unknown> | number[] | number[][] | {
+                [key: string]: number[] | number[][] | {
+                    indices: number[];
+                    values: number[];
+                } | undefined;
+            } | null | undefined;
+            shard_key?: string | number | Record<string, unknown> | null | undefined;
+            order_value?: number | Record<string, unknown> | null | undefined;
+        }[];
+        id: string | number;
+        lookup?: Record<string, unknown> | {
+            id: string | number;
+            payload?: Record<string, unknown> | {
+                [key: string]: unknown;
+            } | null | undefined;
+            vector?: Record<string, unknown> | number[] | number[][] | {
+                [key: string]: number[] | number[][] | {
+                    indices: number[];
+                    values: number[];
+                } | undefined;
+            } | null | undefined;
+            shard_key?: string | number | Record<string, unknown> | null | undefined;
+            order_value?: number | Record<string, unknown> | null | undefined;
+        } | null | undefined;
+    }[]>;
+}

package/knowledge/graph/graph.service.d.ts

@@ -0,0 +1,24 @@
+import { GraphLocalService } from './graph.local.service';
+import { GraphKnolwdgeService } from './graph.knowledge.service';
+import { GraphSplitNodeInput } from './type';
+import { EdgeItemType, EdgePayload, KnowledgeGraphItemType, NodeItemType } from './define/define';
+import { GraphQueryService } from './graph.query.service';
+import { QueryParams } from './const';
+export declare class GraphService {
+    #private;
+    local: GraphLocalService;
+    knowledge: GraphKnolwdgeService;
+    graphExport$$: import("static-injector").Signal<import("graphology-types").SerializedGraph<import("./define").GraphNodeAttr, import("./define").GraphEdgeAttr, import("graphology-types").Attributes>>;
+    createQuery(queryParams: QueryParams): GraphQueryService;
+    loadDataInit$$: import("static-injector").Signal<Promise<void>>;
+    initGraph(): import("graphology").default<import("./define").GraphNodeAttr, import("./define").GraphEdgeAttr, import("graphology-types").Attributes>;
+    getGraph(): import("graphology").default<import("./define").GraphNodeAttr, import("./define").GraphEdgeAttr, import("graphology-types").Attributes>;
+    splitNode(options: GraphSplitNodeInput): Promise<void>;
+    mergeNode(options: GraphSplitNodeInput): Promise<void>;
+    changeNodeDescription(item: NodeItemType): Promise<void>;
+    changeEdge(item: EdgeItemType, oldItem: Pick<EdgePayload, 'id' | 'source' | 'target'>): Promise<void>;
+    add(input: KnowledgeGraphItemType): Promise<void>;
+    deleteNodeItem(item: Pick<NodeItemType, 'id' | 'name'>): Promise<void>;
+    deleteEdge(item: Pick<EdgeItemType, 'id' | 'source' | 'target'>): Promise<void>;
+    deleteNodeByName(name: string): Promise<void>;
+}

package/knowledge/graph/graph.util.service.d.ts

@@ -0,0 +1,31 @@
+export declare class GraphKnowledgeUtilService {
+    #private;
+    updateContext(fn: () => Promise<{
+        upsert?: {
+            nodes?: {
+                id: any;
+                payload: any;
+            }[];
+            edges?: {
+                id: any;
+                payload: any;
+            }[];
+            keywords?: {
+                id: any;
+                payload: any;
+            }[];
+        };
+        delete?: {
+            nodes?: {
+                id: any;
+            }[] | {
+                filter?: any;
+            };
+            edges?: {
+                id: any;
+            }[] | {
+                filter?: any;
+            };
+        };
+    }>): Promise<Promise<(void | void[])[]>>;
+}

package/knowledge/graph/type.d.ts

@@ -0,0 +1,11 @@
+import { FileChunkPayload } from '../common/define/chunk';
+import { EdgeQueryPayload, NodeQueryPayload } from './define';
+export interface QueryContext {
+    nodes: NodeQueryPayload[];
+    edges: EdgeQueryPayload[];
+    chunks: FileChunkPayload[];
+}
+export type GraphSplitNodeInput = {
+    node: string;
+    list: string[];
+};

package/knowledge/graph/util/format-attr.d.ts

@@ -0,0 +1,48 @@
+import { GraphNodeAttr } from '../define/define';
+export declare function getNodeType(list: GraphNodeAttr['list']): string;
+export declare function formatNodeAttr2(list: {
+    data: any;
+    score: number;
+}[]): {
+    list: {
+        kind: "node";
+        id: string;
+        chunkId: string;
+        fileName: string;
+        name: string;
+        description: string;
+        type: string;
+    }[];
+    name: string;
+    type: string;
+    degree: number;
+};
+export type FormatGraphNodeAttr = ReturnType<typeof formatNodeAttr2>;
+export type QueryGraphNodeAttr = FormatGraphNodeAttr & {
+    degree: number;
+};
+export declare function formatEdgeAttr2(list: {
+    data: any;
+    score: number;
+}[]): {
+    list: {
+        chunkId: string;
+        kind: "edge";
+        name: string;
+        fileName: string;
+        id: string;
+        source: string;
+        target: string;
+        description: string;
+        strength: number;
+        keywords?: string[] | undefined;
+    }[];
+    name: string;
+    source: string;
+    target: string;
+    degree: number;
+};
+export type FormatGraphEdgeAttr = ReturnType<typeof formatEdgeAttr2>;
+export type QueryGraphEdgeAttr = FormatGraphEdgeAttr & {
+    degree: number;
+};

package/knowledge/graph/util/graph-util.d.ts

@@ -0,0 +1,5 @@
+import { QueryGraphEdgeAttr, QueryGraphNodeAttr } from './format-attr';
+/** 用来计算权重 */
+export declare function sigmoid(z: number): number;
+export declare function getNodeStrList(item: QueryGraphNodeAttr, index: number): (string | number)[];
+export declare function getEdgeStrList(item: QueryGraphEdgeAttr, i: number): (string | number)[];

package/knowledge/graph/util.d.ts

@@ -0,0 +1 @@
+export declare function getEdgeName(source: string, target: string): string;

package/knowledge/graph/vecotr-format.d.ts

@@ -0,0 +1,11 @@
+export declare function edgeVectorString(options: {
+    description: string;
+    source: string;
+    target: string;
+    keywords?: string[];
+}): string;
+export declare function nodeVectorString(options: {
+    type: string;
+    name: string;
+    description: string;
+}): string;

package/knowledge/index.d.ts

@@ -0,0 +1,17 @@
+export * from './knowledge.manager.service';
+export * from './normal/normal.knowledge.service';
+export * from './dict/dict.knowledge.service';
+export * from './graph/graph.handle.service';
+export * from './graph/graph.knowledge.service';
+export * from './graph/graph.local.service';
+export * from './graph/graph.query.service';
+export * from './graph/graph.service';
+export * from './normal/define';
+export * from './dict/define';
+export * from './graph/define';
+export * from './const';
+export * from './graph/const';
+export * from './type';
+export * from './graph/type';
+export * from './article';
+export * from './common';

package/knowledge/knowledge.manager.service.d.ts

@@ -0,0 +1,42 @@
+import { RootStaticInjectOptions } from 'static-injector';
+import { ImportList, KnowledgeCollectionInput, KnowledgeConfig, KnowledgeType } from './type';
+import { DictKnowledgeService } from './dict/dict.knowledge.service';
+import { GraphKnolwdgeService } from './graph/graph.knowledge.service';
+import type { Provider, R3Injector, Signal } from 'static-injector';
+import { DictInput } from '@shenghuabi/knowledge/file-parser';
+import { NormalKnowledgeService } from './normal/normal.knowledge.service';
+import { GraphService } from './graph/graph.service';
+import { ArticleKnowledgeService } from './article';
+export declare class KnowledgeManagerService extends RootStaticInjectOptions {
+    #private;
+    getConfig(name: string): Promise<Signal<KnowledgeConfig>>;
+    protected getInjector(name: string, extraProviders?: Provider[]): Promise<R3Injector>;
+    create(name: string, collection: KnowledgeCollectionInput): Promise<void>;
+    /** 普通知识库和图谱知识库用 */
+    importFiles(name: string, filePathList: string[], signal?: AbortSignal): Promise<void>;
+    /** 字典专用 */
+    importDict(name: string, input: DictInput): Promise<number>;
+    get(name: string): Promise<ArticleKnowledgeService | NormalKnowledgeService | DictKnowledgeService | GraphKnolwdgeService>;
+    importTextFile(name: string, dir: string, filePathList: string[], signal?: AbortSignal): Promise<void>;
+    deleteItem(name: string, fileName: string): Promise<void>;
+    updateItem(name: string, fileName: string, content: string): Promise<void>;
+    /** 通用接口 */
+    addCollection(name: string, collection: KnowledgeCollectionInput): Promise<void>;
+    /** 通用接口 */
+    deleteCollection(name: string, collectionName: string): Promise<boolean>;
+    changeActivateCollection(name: string, collectionName: string): Promise<void>;
+    destroy(name: string): Promise<void>;
+    export(name: string): Promise<{
+        collection: string;
+        name?: string | undefined;
+        creation_time?: string | null | undefined;
+        size?: number | undefined;
+        checksum?: string | null | undefined;
+    }[]>;
+    import(name: string, options: {
+        snapshotList: ImportList;
+        activateCollection: string;
+        type: KnowledgeType;
+    }): Promise<void>;
+    getGraph(name: string): Promise<GraphService>;
+}

package/knowledge/knowledge.util.service.d.ts

@@ -0,0 +1,21 @@
+import { RootStaticInjectOptions } from 'static-injector';
+import { QdrantClient } from '@qdrant/qdrant-js';
+import { CacheQueue } from '@shenghuabi/knowledge/util';
+import { NormalizeFs } from '@cyia/vfs2';
+export declare class KnowledgeUtilService extends RootStaticInjectOptions {
+    #private;
+    multiDelete(list: string[], filter: Parameters<QdrantClient['delete']>[1]): Promise<void[]>;
+    updatePointsQueue<Vector extends Record<string, number[]>, Payload extends Record<string, any>>(collectionName: string): CacheQueue<{
+        id: string | number;
+        vector: Vector;
+        payload: Payload;
+    }>;
+    destroyKnowledge(list: string[], vfs?: NormalizeFs): Promise<void>;
+    export(list: string[]): Promise<{
+        collection: string;
+        name?: string | undefined;
+        creation_time?: string | null | undefined;
+        size?: number | undefined;
+        checksum?: string | null | undefined;
+    }[]>;
+}

package/knowledge/normal/define/config.d.ts

@@ -0,0 +1,60 @@
+import * as v from 'valibot';
+export declare const NormalCollectionDefine: v.ObjectSchema<{
+    readonly collectionName: v.StringSchema<undefined>;
+    readonly embeddingTemplate: v.OptionalSchema<v.ObjectSchema<{
+        readonly entry: v.OptionalSchema<v.ObjectSchema<{
+            readonly enable: v.BooleanSchema<undefined>;
+            readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+        }, undefined>, undefined>;
+    }, undefined>, undefined>;
+    readonly size: v.NumberSchema<undefined>;
+}, undefined>;
+export declare const NormalKnowledgeConfigDefine: v.SchemaWithPipe<readonly [v.ObjectSchema<{
+    readonly type: v.OptionalSchema<v.LiteralSchema<"normal", undefined>, "normal">;
+    readonly collectionList: v.ArraySchema<v.ObjectSchema<{
+        readonly collectionName: v.StringSchema<undefined>;
+        readonly embeddingTemplate: v.OptionalSchema<v.ObjectSchema<{
+            readonly entry: v.OptionalSchema<v.ObjectSchema<{
+                readonly enable: v.BooleanSchema<undefined>;
+                readonly value: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
+            }, undefined>, undefined>;
+        }, undefined>, undefined>;
+        readonly size: v.NumberSchema<undefined>;
+    }, undefined>, undefined>;
+    readonly name: v.StringSchema<undefined>;
+    readonly activateCollection: v.StringSchema<undefined>;
+}, undefined>, v.TransformAction<{
+    type: "normal";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}, {
+    /** 激活的普通知识库(文件切片) */
+    activateName: string;
+    type: "normal";
+    collectionList: {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    }[];
+    name: string;
+    activateCollection: string;
+}>]>;
+export type NormalKnowledgeConfig = v.InferInput<typeof NormalKnowledgeConfigDefine>;
+export type NormalKnowledgeConfigInline = v.InferOutput<typeof NormalKnowledgeConfigDefine>;
+export type NormalCollectionInput = v.InferInput<typeof NormalCollectionDefine>;
+export type NormalCollectionInlineType = v.InferOutput<typeof NormalCollectionDefine>;

package/knowledge/normal/define/index.d.ts

@@ -0,0 +1 @@
+export * from './config';

package/knowledge/normal/normal.knowledge.service.d.ts

@@ -0,0 +1,49 @@
+import { NormalCollectionInlineType } from './define/config';
+import { FileChunkPayload } from '../common/define/chunk';
+import { CommonKnowledgeService } from '../common/common.knowledge.service';
+export declare class NormalKnowledgeService extends CommonKnowledgeService {
+    #private;
+    protected KeyWordIndex: string[];
+    protected getPayload(fileName: string, content: string): Record<string, any>;
+    formatCollection(input: any): {
+        collectionName: string;
+        embeddingTemplate?: {
+            entry?: {
+                enable: boolean;
+                value?: string | undefined;
+            } | undefined;
+        } | undefined;
+        size: number;
+    };
+    protected createCollection(collection: NormalCollectionInlineType): Promise<void>;
+    /** 创建知识库 */
+    create(collection: NormalCollectionInlineType): Promise<void>;
+    insertItemOnly(fileName: string, content: string, collectionList: NormalCollectionInlineType[]): Promise<{
+        id: string;
+        vector: {
+            chunk: number[];
+        };
+        payload: FileChunkPayload;
+    }[][]>;
+    protected _insertItem(fileName: string, content: string): Promise<false | {
+        id: string;
+        vector: {
+            chunk: number[];
+        };
+        payload: FileChunkPayload;
+    }[][]>;
+    insertItem(fileName: string, content: string, signal?: AbortSignal): Promise<void>;
+    deleteItem(fileName: string): Promise<void>;
+    updateItem(fileName: string, content: string): Promise<void>;
+    addCollection(collection: NormalCollectionInlineType): Promise<void>;
+    deleteCollection(collectionName: string): Promise<boolean>;
+    changeActivateCollection(collectionName: string): Promise<void>;
+    export(): Promise<{
+        collection: string;
+        name?: string | undefined;
+        creation_time?: string | null | undefined;
+        size?: number | undefined;
+        checksum?: string | null | undefined;
+    }[]>;
+    destroy(): Promise<void>;
+}

package/knowledge/template.format.d.ts

@@ -0,0 +1,6 @@
+export declare function interpolate(input: string, value: Record<string, any>): string;
+/** 词条格式化 */
+export declare function entryFormat(payload: Record<string, any>, knowledge: string, defaultContent: string, embedingTemplate?: {
+    value?: string;
+    enable: boolean;
+}): string;

package/knowledge/type.d.ts

@@ -0,0 +1,28 @@
+import { GraphCollectionInlineType, GraphCollectionInput, GraphKnowledgeConfig } from './graph/define/config';
+import { NormalCollectionInlineType, NormalCollectionInput, NormalKnowledgeConfig } from './normal/define/config';
+import { DictCollectionInlineType, DictCollectionInput, DictKnowledgeConfig } from './dict/define/config';
+import * as v from 'valibot';
+import { KnowledgeConfigDefine } from './define';
+import { ArticleKnowledgeConfig } from './article';
+export type KnowledgeConfigInput = NormalKnowledgeConfig | GraphKnowledgeConfig | DictKnowledgeConfig | ArticleKnowledgeConfig;
+export type KnowledgeConfig = v.InferOutput<typeof KnowledgeConfigDefine>;
+export type TextSplitter = (content: string, metadata: Record<string, any>, collectionName: string) => Promise<{
+    pageContent: string;
+    metadata: Record<string, any>;
+}[]>;
+export type OCR = (data: string) => Promise<string>;
+export type Text2Vec = <T extends string | string[]>(value: T, 
+/** 集合名字 */ collectionName: string) => Promise<T extends string ? number[] : number[][]>;
+export type Log = {
+    info: (...args: any) => void;
+    warn: (...args: any) => void;
+    error: (...args: any) => void;
+};
+export type KnowledgeCollection = NormalCollectionInlineType | DictCollectionInlineType | GraphCollectionInlineType;
+export type KnowledgeCollectionInput = NormalCollectionInput | DictCollectionInput | GraphCollectionInput;
+export type ImportList = {
+    collection: string;
+    checksum?: string | null;
+    filePath: string;
+}[];
+export type KnowledgeType = NonNullable<KnowledgeConfigInput['type']>;