@magda/semantic-indexer-sdk 6.0.0-alpha.2

package/dist/index.d.ts

@@ -0,0 +1,238 @@
+import { ResponseBody } from '@opensearch-project/opensearch/api/_types/_core.search.js';
+import urijs from 'urijs';
+
+declare interface ApiClientBaseConfig {
+    baseApiUrl?: string;
+}
+
+declare interface ApiKeyBasedConfigOptions extends ApiClientBaseConfig {
+    apiKeyId?: string;
+    apiKey?: string;
+}
+
+declare abstract class BaseApiClient {
+    readonly authMode: "jwtToken" | "apiKey" | "noAuth";
+    protected baseApiUrl: string;
+    private readonly baseApiUri;
+    protected jwtSecret: string;
+    readonly apiKeyId: string;
+    protected apiKey: string;
+    readonly userId: string;
+    constructor(options: BaseApiClientConfig);
+    protected getBaseApiUri(): urijs;
+    protected setHeader(headers: HeadersInit | undefined | null, headerName: string, headerValue: string): Record<string, string> | [string, string][] | Headers;
+    protected addAuthHeader(config?: RequestInit): {
+        body?: BodyInit;
+        cache?: RequestCache;
+        credentials?: RequestCredentials;
+        headers?: HeadersInit;
+        integrity?: string;
+        keepalive?: boolean;
+        method?: string;
+        mode?: RequestMode;
+        redirect?: RequestRedirect;
+        referrer?: string;
+        referrerPolicy?: ReferrerPolicy;
+        signal?: AbortSignal;
+        window?: null;
+    };
+}
+
+declare type BaseApiClientConfig = JWTBasedConfigOptions & ApiKeyBasedConfigOptions;
+
+declare class Chunker {
+    private strategy;
+    constructor(strategy: ChunkStrategy);
+    chunk(text: string): Promise<ChunkResult[]>;
+}
+
+export declare interface ChunkResult {
+    text: string;
+    position: number;
+    length: number;
+    overlap: number;
+}
+
+declare interface ChunkStrategy {
+    chunk: ChunkStrategyType;
+}
+
+export declare type ChunkStrategyType = (text: string) => Promise<ChunkResult[]>;
+
+export declare function commonYargs(defaultPort: number, defaultInternalUrl: string): SemanticIndexerArguments;
+
+export declare type CreateEmbeddingText = (params: CreateEmbeddingTextParams) => Promise<EmbeddingText>;
+
+export declare type CreateEmbeddingTextParams = {
+    record: Record_2;
+    format: string | null;
+    filePath: string | null;
+    url: string | null;
+};
+
+declare class EmbeddingApiClient extends BaseApiClient {
+    private taskSize;
+    constructor(options: EmbeddingApiClientConfig);
+    private testConnection;
+    get(text: string): Promise<number[]>;
+    get(textList: string[]): Promise<number[][]>;
+}
+
+declare interface EmbeddingApiClientConfig extends BaseApiClientConfig {
+}
+
+declare interface EmbeddingApiConfig {
+    baseUrl: string;
+    bulkEmbeddingsSize: number;
+}
+
+export declare type EmbeddingText = {
+    text: string;
+    subObjects?: Array<{
+        subObjectId?: string;
+        subObjectType?: string;
+        text: string;
+    }>;
+};
+
+export declare function indexEmbeddingText(options: SemanticIndexerOptions, EmbeddingText: EmbeddingText, meta: {
+    recordId: string;
+    fileFormat?: string;
+}, chunker: Chunker, embeddingApiClient: EmbeddingApiClient, opensearchApiClient: OpensearchApiClient): Promise<void>;
+
+export declare type ItemType = "registryRecord" | "storageObject";
+
+declare interface JWTBasedConfigOptions extends ApiClientBaseConfig {
+    jwtSecret?: string;
+    userId?: string;
+}
+
+declare interface MinioConfig {
+    endPoint: string;
+    port: number;
+    useSSL: boolean;
+    accessKey: string;
+    secretKey: string;
+    region: string;
+    defaultDatasetBucket: string;
+}
+
+declare type MinionArguments = {
+    listenPort: string | number;
+    internalUrl: string;
+    jwtSecret: string;
+    userId: string;
+    registryUrl: string;
+    enableMultiTenant: boolean;
+    tenantUrl: string;
+    retries: string | number;
+    crawlerRecordFetchNumber?: string | number;
+};
+
+declare class OpensearchApiClient {
+    private client;
+    constructor(config: OpensearchConfig_2);
+    static getOpensearchApiClient(openSearchConfig: OpensearchConfig_2): Promise<OpensearchApiClient>;
+    createIndex(indexDefinition: {
+        indexName: string;
+        settings?: any;
+        mappings?: any;
+    }): Promise<void>;
+    deleteIndex(indexName: string): Promise<void>;
+    indexExists(indexName: string): Promise<boolean>;
+    indexDocument(indexName: string, document: any): Promise<void>;
+    bulkIndexDocument(indexName: string, documents: any[]): Promise<void>;
+    searchDocuments(indexName: string, query: string): Promise<ResponseBody>;
+    deleteDocument(indexName: string, documentId: string): Promise<void>;
+}
+
+declare interface OpensearchConfig {
+    serverUrl: string;
+    bulkIndexSize: number;
+    indices: {
+        semanticIndex: {
+            indexName: string;
+            indexVersion: number;
+            numberOfShards: number;
+            numberOfReplicas: number;
+            knnVectorFieldConfig: {
+                mode: string;
+                dimension: number;
+                spaceType: string;
+                efConstruction: number;
+                efSearch: number;
+                m: number;
+                encoder?: {
+                    name: string;
+                    type: string;
+                    clip: boolean;
+                };
+                compressionLevel?: string;
+            };
+        };
+    };
+}
+
+declare interface OpensearchConfig_2 {
+    url?: string;
+}
+
+/**
+ * A record in the registry, usually including data for one or more aspects, unique for a tenant.
+ */
+declare class Record_2 {
+    /**
+     * The identifier of the record
+     */
+    "id": string;
+    /**
+     * The name of the record
+     */
+    "name": string;
+    /**
+     * The aspects included in this record
+     */
+    "aspects": any;
+    /**
+     * A tag representing the action by the source of this record (e.g. an id for a individual crawl of a data portal).
+     */
+    "sourceTag": string;
+    /**
+     * The identifier of a tenant
+     */
+    "tenantId": number;
+}
+export { Record_2 as Record }
+
+declare function semanticIndexer(userConfig: SemanticIndexerOptions): Promise<void>;
+export default semanticIndexer;
+
+declare interface SemanticIndexerArguments extends MinionArguments {
+    semanticIndexerConfig: SemanticIndexerConfig;
+    minioConfig: MinioConfig;
+}
+
+declare interface SemanticIndexerConfig {
+    semanticIndexer: {
+        chunkSizeLimit: number;
+        overlap: number;
+        opensearch: OpensearchConfig;
+        embeddingApi: EmbeddingApiConfig;
+    };
+}
+
+export declare interface SemanticIndexerOptions {
+    argv: SemanticIndexerArguments;
+    id: string;
+    itemType: ItemType;
+    aspects?: string[];
+    optionalAspects?: string[];
+    formatTypes?: string[];
+    createEmbeddingText: CreateEmbeddingText;
+    chunkStrategy?: ChunkStrategyType;
+    chunkSizeLimit?: number;
+    overlap?: number;
+    autoDownloadFile?: boolean;
+}
+
+export { }