@shenghuabi/knowledge 1.0.21

package/embedding/embedding.service.d.ts

@@ -0,0 +1,4 @@
+export declare class EmbeddingService {
+    #private;
+    text2Vec<T extends string | string[]>(value: T): Promise<T extends string ? number[] : number[][]>;
+}

package/embedding/index.d.ts

@@ -0,0 +1,2 @@
+export * from './embedding.service';
+export * from './type';

package/embedding/type.d.ts

@@ -0,0 +1,12 @@
+import { InjectionToken } from 'static-injector';
+export type EmbeddingConfig = {
+    /** 缓存嵌入数量 */
+    maxCache: number;
+    /** 超时清除 */
+    ttl: number;
+    /** 每批最大数量 */
+    maxBatchSize: number;
+    maxAsyncCount: number;
+    text2Vec: (list: string[]) => Promise<number[][]>;
+};
+export declare const EmbeddingConfigToken: InjectionToken<EmbeddingConfig>;

package/embedding.mjs

@@ -0,0 +1,61 @@
+// packages/embedding/embedding.service.ts
+import { inject } from "static-injector";
+
+// packages/embedding/type.ts
+import { InjectionToken } from "static-injector";
+var EmbeddingConfigToken = new InjectionToken(
+  "EmbeddingConfig"
+);
+
+// packages/embedding/embedding.service.ts
+import { LRUCache } from "lru-cache";
+import { chunk, uniq } from "es-toolkit";
+import { promise as fastq } from "fastq";
+var EmbeddingService = class {
+  #config = inject(EmbeddingConfigToken);
+  #cache = new LRUCache({
+    max: this.#config.maxCache,
+    ttl: this.#config.ttl
+  });
+  #queue = fastq(
+    async (value) => this.#config.text2Vec(value),
+    this.#config.maxAsyncCount
+  );
+  async text2Vec(value) {
+    if (typeof value === "string") {
+      if (!this.#cache.has(value)) {
+        const data = this.#queue.push([value]).then(([item]) => item);
+        this.#cache.set(value, data);
+      }
+      return await this.#cache.get(value);
+    }
+    const cachedObj = {};
+    const pendingList = [];
+    for (const item of uniq(value)) {
+      if (this.#cache.has(item)) {
+        cachedObj[item] = this.#cache.get(item);
+      } else {
+        pendingList.push(item);
+      }
+    }
+    if (pendingList.length) {
+      const chunPendingList = chunk(pendingList, this.#config.maxBatchSize);
+      await Promise.all(
+        chunPendingList.map(
+          (subList) => this.#queue.push(subList).then(
+            (list) => list.forEach((item, i) => {
+              cachedObj[subList[i]] = Promise.resolve(item);
+              this.#cache.set(subList[i], cachedObj[subList[i]]);
+            })
+          )
+        )
+      );
+    }
+    return Promise.all(value.map((item) => cachedObj[item]));
+  }
+};
+export {
+  EmbeddingConfigToken,
+  EmbeddingService
+};
+//# sourceMappingURL=embedding.mjs.map

package/embedding.mjs.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../packages/embedding/embedding.service.ts", "../packages/embedding/type.ts"],
+  "sourcesContent": ["import { inject } from 'static-injector';\nimport { EmbeddingConfigToken } from './type';\nimport { LRUCache } from 'lru-cache';\nimport { chunk, uniq } from 'es-toolkit';\nimport { promise as fastq } from 'fastq';\nexport class EmbeddingService {\n  #config = inject(EmbeddingConfigToken);\n  #cache = new LRUCache<string, Promise<number[]>>({\n    max: this.#config.maxCache,\n    ttl: this.#config.ttl,\n  });\n\n  #queue = fastq(\n    async (value: string[]): Promise<number[][]> =>\n      this.#config.text2Vec(value),\n    this.#config.maxAsyncCount,\n  );\n  async text2Vec<T extends string | string[]>(\n    value: T,\n  ): Promise<T extends string ? number[] : number[][]> {\n    if (typeof value === 'string') {\n      if (!this.#cache.has(value)) {\n        const data = this.#queue.push([value]).then(([item]) => item);\n        this.#cache.set(value, data);\n      }\n      return (await this.#cache.get(value))! as any;\n    }\n    const cachedObj = {} as Record<string, Promise<number[]>>;\n    const pendingList = [] as string[];\n    for (const item of uniq(value)) {\n      if (this.#cache.has(item)) {\n        cachedObj[item] = this.#cache.get(item)!;\n      } else {\n        pendingList.push(item);\n      }\n    }\n    if (pendingList.length) {\n      const chunPendingList = chunk(pendingList, this.#config.maxBatchSize);\n      await Promise.all(\n        chunPendingList.map((subList) =>\n          this.#queue.push(subList).then((list) =>\n            list.forEach((item, i) => {\n              cachedObj[subList[i]] = Promise.resolve(item);\n              this.#cache.set(subList[i], cachedObj[subList[i]]);\n            }),\n          ),\n        ),\n      );\n    }\n    return Promise.all(value.map((item) => cachedObj[item]!)) as any;\n  }\n}\n", "import { InjectionToken } from 'static-injector';\n\nexport type EmbeddingConfig = {\n  /** 缓存嵌入数量 */\n  maxCache: number;\n  /** 超时清除 */\n  ttl: number;\n  /** 每批最大数量 */\n  maxBatchSize: number;\n  maxAsyncCount: number;\n  text2Vec: (list: string[]) => Promise<number[][]>;\n};\nexport const EmbeddingConfigToken = new InjectionToken<EmbeddingConfig>(\n  'EmbeddingConfig',\n);\n"],
+  "mappings": ";AAAA,SAAS,cAAc;;;ACAvB,SAAS,sBAAsB;AAYxB,IAAM,uBAAuB,IAAI;AAAA,EACtC;AACF;;;ADZA,SAAS,gBAAgB;AACzB,SAAS,OAAO,YAAY;AAC5B,SAAS,WAAW,aAAa;AAC1B,IAAM,mBAAN,MAAuB;AAAA,EAC5B,UAAU,OAAO,oBAAoB;AAAA,EACrC,SAAS,IAAI,SAAoC;AAAA,IAC/C,KAAK,KAAK,QAAQ;AAAA,IAClB,KAAK,KAAK,QAAQ;AAAA,EACpB,CAAC;AAAA,EAED,SAAS;AAAA,IACP,OAAO,UACL,KAAK,QAAQ,SAAS,KAAK;AAAA,IAC7B,KAAK,QAAQ;AAAA,EACf;AAAA,EACA,MAAM,SACJ,OACmD;AACnD,QAAI,OAAO,UAAU,UAAU;AAC7B,UAAI,CAAC,KAAK,OAAO,IAAI,KAAK,GAAG;AAC3B,cAAM,OAAO,KAAK,OAAO,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,MAAM,IAAI;AAC5D,aAAK,OAAO,IAAI,OAAO,IAAI;AAAA,MAC7B;AACA,aAAQ,MAAM,KAAK,OAAO,IAAI,KAAK;AAAA,IACrC;AACA,UAAM,YAAY,CAAC;AACnB,UAAM,cAAc,CAAC;AACrB,eAAW,QAAQ,KAAK,KAAK,GAAG;AAC9B,UAAI,KAAK,OAAO,IAAI,IAAI,GAAG;AACzB,kBAAU,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI;AAAA,MACxC,OAAO;AACL,oBAAY,KAAK,IAAI;AAAA,MACvB;AAAA,IACF;AACA,QAAI,YAAY,QAAQ;AACtB,YAAM,kBAAkB,MAAM,aAAa,KAAK,QAAQ,YAAY;AACpE,YAAM,QAAQ;AAAA,QACZ,gBAAgB;AAAA,UAAI,CAAC,YACnB,KAAK,OAAO,KAAK,OAAO,EAAE;AAAA,YAAK,CAAC,SAC9B,KAAK,QAAQ,CAAC,MAAM,MAAM;AACxB,wBAAU,QAAQ,CAAC,CAAC,IAAI,QAAQ,QAAQ,IAAI;AAC5C,mBAAK,OAAO,IAAI,QAAQ,CAAC,GAAG,UAAU,QAAQ,CAAC,CAAC,CAAC;AAAA,YACnD,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,WAAO,QAAQ,IAAI,MAAM,IAAI,CAAC,SAAS,UAAU,IAAI,CAAE,CAAC;AAAA,EAC1D;AACF;",
+  "names": []
+}

package/file-parser/const.d.ts

@@ -0,0 +1,20 @@
+import { FileTypeResult } from 'file-type';
+import { InjectionToken, Signal } from 'static-injector';
+import { FormatedData } from './document-file-parser.service';
+export interface FileParser {
+    priority: number;
+    parse: (fileName: string, buffer: Uint8Array | ArrayBuffer, type: FileTypeResult | undefined) => Promise<FormatedData[] | undefined>;
+}
+export declare const FileParserToken: InjectionToken<FileParser[]>;
+export declare const ImageParserToken: InjectionToken<(assetPath: string, prefix: string, buffer: Buffer<ArrayBufferLike>) => Promise<{
+    content: string;
+    parseTo: string;
+}>>;
+export declare const DocumentParserConfigToken: InjectionToken<Signal<{
+    pdfAsImage?: {
+        enable?: boolean;
+        viewPortOptions?: {
+            scale?: number;
+        };
+    };
+}>>;

package/file-parser/dict/dict-format/dsl/dsl-parse.service.d.ts

@@ -0,0 +1,6 @@
+import { RootStaticInjectOptions } from 'static-injector';
+import { AbstractDictParse, DictParseResult } from '../../type';
+import { PassCode } from '@cyia/mdict-reader';
+export declare class DslParseService extends RootStaticInjectOptions implements AbstractDictParse {
+    parse(filePath: string, options?: Partial<PassCode>): Promise<DictParseResult>;
+}

package/file-parser/dict/dict-format/dsl/dsl.format.d.ts

@@ -0,0 +1 @@
+export declare function dslFormat(input: string): string;

package/file-parser/dict/dict-format/mdict-parse.service.d.ts

@@ -0,0 +1,20 @@
+import { AbstractDictParse } from '../type';
+import { Mdict, PassCode } from '@cyia/mdict-reader';
+import { RootStaticInjectOptions } from 'static-injector';
+export declare class MdictParseService extends RootStaticInjectOptions implements AbstractDictParse {
+    parse(filePathList: string, options?: PassCode): Promise<{
+        info: {
+            name: string;
+            fileName: string;
+        };
+        dataListGenerator: () => AsyncGenerator<{
+            content: string;
+            word: string;
+        }, void, unknown>;
+        afterSave: (assetFolder: string) => Promise<void>;
+    }>;
+    wordListGenerator(mdxInstance: Mdict): AsyncGenerator<{
+        content: string;
+        word: string;
+    }, void, unknown>;
+}

package/file-parser/dict/dict-format/stardict-parse.service.d.ts

@@ -0,0 +1,12 @@
+import { AbstractDictParse } from '../type';
+import { RootStaticInjectOptions } from 'static-injector';
+export declare class StardictParseService extends RootStaticInjectOptions implements AbstractDictParse {
+    #private;
+    parse(filePathList: string): Promise<{
+        info: any;
+        dataListGenerator: () => AsyncGenerator<{
+            word: string;
+            content: string;
+        }, void, unknown>;
+    }>;
+}

package/file-parser/dict/dict-format/yaml-parse.service.d.ts

@@ -0,0 +1,25 @@
+import { AbstractDictParse } from '../type';
+import { RootStaticInjectOptions } from 'static-injector';
+import * as v from 'valibot';
+export declare const YamlDefine: v.ObjectSchema<{
+    readonly list: v.ArraySchema<v.ObjectSchema<{
+        readonly word: v.StringSchema<undefined>;
+        readonly content: v.StringSchema<undefined>;
+        readonly extra: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.AnySchema, undefined>, undefined>;
+    }, undefined>, undefined>;
+}, undefined>;
+export declare class YamlDictParseService extends RootStaticInjectOptions implements AbstractDictParse {
+    parse(filePathList: string): Promise<{
+        info: {
+            fileName: string;
+            name: string;
+        };
+        dataListGenerator: () => AsyncGenerator<{
+            word: string;
+            content: string;
+            extra?: {
+                [x: string]: any;
+            } | undefined;
+        }, void, unknown>;
+    }>;
+}

package/file-parser/dict/dict.service.d.ts

@@ -0,0 +1,10 @@
+import { DictInput } from './type';
+import { RootStaticInjectOptions } from 'static-injector';
+export declare class DictService extends RootStaticInjectOptions {
+    #private;
+    getDictName(input: DictInput): Promise<string>;
+    /**
+     * 第一个需要修改为3个操作
+     */
+    importDict(name: string, dir: string, input: DictInput): Promise<AsyncGenerator<import("./type").WordItem, any, any>>;
+}

package/file-parser/dict/index.d.ts

@@ -0,0 +1,2 @@
+export * from './dict.service';
+export * from './type';

package/file-parser/dict/type.d.ts

@@ -0,0 +1,24 @@
+import { PassCode } from '@cyia/mdict-reader';
+export interface AbstractDictParse {
+    parse(filePath: string, options?: Partial<PassCode>): Promise<DictParseResult>;
+}
+export interface WordItem {
+    word: string;
+    content: string;
+    extra?: Record<string, any>;
+    htmlContent?: string;
+}
+export interface DictParseResult {
+    info: {
+        name: string;
+        fileName: string | null;
+        [name: string]: any;
+    };
+    dataListGenerator: () => AsyncGenerator<WordItem>;
+    afterSave?: (assetPath: string) => Promise<void>;
+}
+export type DictInput = {
+    filePath: string;
+    /** todo 可以加yaml */
+    type: 'stardict' | 'mdict' | 'yaml' | 'dsl';
+} & Partial<PassCode>;

package/file-parser/document-file-parser.service.d.ts

@@ -0,0 +1,15 @@
+import { FileTypeResult } from 'file-type';
+import { RootStaticInjectOptions } from 'static-injector';
+import type { Document } from '@langchain/core/documents';
+export type ParsedData = Document<Record<string, any>> & {
+    parseTo?: string;
+};
+export type FormatedData = {
+    title: any;
+    content: string;
+    parseTo?: string;
+};
+export declare class DocumentFileParserService extends RootStaticInjectOptions {
+    #private;
+    parse(filePath: string, buffer: Uint8Array | ArrayBuffer, type: FileTypeResult | undefined): Promise<FormatedData[] | undefined>;
+}

package/file-parser/document-loader/pdf-img.loader.d.ts

@@ -0,0 +1,8 @@
+import { Injector } from 'static-injector';
+export declare function pdfImageLoader(buffer: Uint8Array | ArrayBuffer, filePath: string, injector: Injector): Promise<{
+    pageContent: string;
+    metadata: {
+        title: any;
+    };
+    parseTo: string;
+}[]>;

package/file-parser/document-loader/xlsx.loader.d.ts

@@ -0,0 +1,6 @@
+export declare function xlsxLoader(buffer: Uint8Array | ArrayBuffer): Promise<{
+    pageContent: string;
+    metadata: {
+        title: string;
+    };
+}[]>;

package/file-parser/file-parser.service.d.ts

@@ -0,0 +1,13 @@
+import { RootStaticInjectOptions } from 'static-injector';
+import { FileParser } from './const';
+import { FormatedData } from './document-file-parser.service';
+export declare class FileParserService extends RootStaticInjectOptions implements FileParser {
+    #private;
+    priority: number;
+    parse(fileName: string, buffer: Uint8Array | ArrayBuffer): Promise<FormatedData[]>;
+    /** 用于支持工作流读文件 */
+    parseOne(fileName: string, buffer: Uint8Array | ArrayBuffer): Promise<{
+        content: string;
+        parseTo: string | undefined;
+    }>;
+}

package/file-parser/index.d.ts

@@ -0,0 +1,6 @@
+export * from './file-parser.service';
+export * from './dict';
+export * from './const';
+export * from './document-file-parser.service';
+export * from './text-parser';
+export * from './vl-parser/markdown.parser';

package/file-parser/text-analyse.d.ts

@@ -0,0 +1 @@
+export declare function textAnalyse(buffer: Uint8Array): import("chardet").EncodingName;

package/file-parser/text-parser.d.ts

@@ -0,0 +1,3 @@
+export declare function bufferDecodeToText(buffer: Uint8Array, metadata?: {
+    path?: string;
+}): string;

package/file-parser/vl-parser/markdown.parser.d.ts

@@ -0,0 +1,8 @@
+type Position = [number, number, number, number];
+export declare function vlMarkdownParser(content: string, options: {
+    imageGet: (type: string, position: Position) => Promise<{
+        src: string;
+        title: string;
+    }>;
+}): Promise<string>;
+export {};