@magda/semantic-indexer-framework 5.2.0

package/dist/indexSchema.d.ts

@@ -0,0 +1,105 @@
+import { SemanticIndexerOptions } from "./index.js";
+export declare function createSemanticIndexerMapping(config: SemanticIndexerOptions): {
+  indexName: string;
+  settings: {
+    index: {
+      number_of_shards: number;
+      number_of_replicas: number;
+    };
+    "index.knn": boolean;
+  };
+  mappings: {
+    properties: {
+      itemType: {
+        type: string;
+      };
+      recordId: {
+        type: string;
+      };
+      aspectId: {
+        type: string;
+      };
+      parentRecordId: {
+        type: string;
+      };
+      fileFormat: {
+        type: string;
+      };
+      subObjectId: {
+        type: string;
+      };
+      subObjectType: {
+        type: string;
+      };
+      index_text_chunk: {
+        type: string;
+      };
+      embedding: {
+        method: {
+          name: string;
+          engine: string;
+          parameters: {
+            encoder?: {
+              name: string;
+              parameters: {
+                type: string;
+                clip: boolean;
+              };
+            };
+            m: number;
+            ef_construction: number;
+            ef_search: number;
+          };
+        };
+        compression_level?: string;
+        type: string;
+        dimension: number;
+        space_type: string;
+        mode: string;
+      };
+      only_one_index_text_chunk: {
+        type: string;
+      };
+      index_text_chunk_length: {
+        type: string;
+      };
+      index_text_chunk_position: {
+        type: string;
+      };
+      index_text_chunk_overlap: {
+        type: string;
+      };
+      indexerId: {
+        type: string;
+      };
+      createTime: {
+        type: string;
+        format: string;
+      };
+      updateTime: {
+        type: string;
+        format: string;
+      };
+    };
+  };
+};
+export type ItemType = "registryRecord" | "storageObject";
+export interface SemanticIndexDocument {
+  itemType: ItemType;
+  recordId: string;
+  aspectId?: string;
+  parentRecordId?: string;
+  fileFormat?: string;
+  subObjectId?: string;
+  subObjectType?: string;
+  index_text_chunk: string;
+  embedding: number[];
+  only_one_index_text_chunk: boolean;
+  index_text_chunk_length: number;
+  index_text_chunk_position: number;
+  index_text_chunk_overlap: number;
+  indexerId: string;
+  createTime: string;
+  updateTime: string;
+}
+export declare function buildSemanticIndexDocument(params: SemanticIndexDocument): SemanticIndexDocument;

package/dist/indexSchema.js

@@ -0,0 +1,98 @@
+const SEMANTIC_INDEX_VERSION = 1;
+export function createSemanticIndexerMapping(config) {
+  const indexConfig = config.argv.semanticIndexerConfig;
+  const knnVectorFieldConfig = indexConfig.knnVectorFieldConfig;
+  if (indexConfig.indexVersion !== SEMANTIC_INDEX_VERSION) {
+    throw new Error(`Index version mismatch. Expected ${SEMANTIC_INDEX_VERSION}, got ${indexConfig.indexVersion}`);
+  }
+  if (indexConfig.knnVectorFieldConfig.compressionLevel &&
+  indexConfig.knnVectorFieldConfig.encoder) {
+    throw new Error("compressionLevel and encoder cannot be used together");
+  }
+  return {
+    indexName: indexConfig.fullIndexName,
+    settings: {
+      index: {
+        number_of_shards: indexConfig.numberOfShards,
+        number_of_replicas: indexConfig.numberOfReplicas
+      },
+      "index.knn": true
+    },
+    mappings: {
+      properties: {
+        itemType: { type: "keyword" },
+        recordId: { type: "keyword" },
+        aspectId: { type: "keyword" },
+        parentRecordId: { type: "keyword" },
+        fileFormat: { type: "keyword" },
+        subObjectId: { type: "keyword" },
+        subObjectType: { type: "keyword" },
+        index_text_chunk: { type: "keyword" },
+        embedding: {
+          type: "knn_vector",
+          dimension: knnVectorFieldConfig.dimension,
+          space_type: knnVectorFieldConfig.spaceType,
+          mode: knnVectorFieldConfig.mode,
+          ...(knnVectorFieldConfig.compressionLevel ?
+          {
+            compression_level: knnVectorFieldConfig.compressionLevel
+          } :
+          {}),
+          method: {
+            name: "hnsw",
+            engine: "faiss",
+            parameters: {
+              m: knnVectorFieldConfig.m,
+              ef_construction: knnVectorFieldConfig.efConstruction,
+              ef_search: knnVectorFieldConfig.efSearch,
+              ...(knnVectorFieldConfig.encoder ?
+              {
+                encoder: {
+                  name: knnVectorFieldConfig.encoder.name,
+                  parameters: {
+                    type: knnVectorFieldConfig.encoder.
+                    type,
+                    clip: knnVectorFieldConfig.encoder.
+                    clip
+                  }
+                }
+              } :
+              {})
+            }
+          }
+        },
+        only_one_index_text_chunk: { type: "boolean" },
+        index_text_chunk_length: { type: "integer" },
+        index_text_chunk_position: { type: "integer" },
+        index_text_chunk_overlap: { type: "integer" },
+        indexerId: { type: "keyword" },
+        createTime: { type: "date", format: "strict_date_time" },
+        updateTime: { type: "date", format: "strict_date_time" }
+      }
+    }
+  };
+}
+export function buildSemanticIndexDocument(params) {
+  return {
+    itemType: params.itemType,
+    recordId: params.recordId,
+    ...(params.aspectId ? { aspectId: params.aspectId } : {}),
+    ...(params.parentRecordId ?
+    { parentRecordId: params.parentRecordId } :
+    {}),
+    ...(params.fileFormat ? { fileFormat: params.fileFormat } : {}),
+    ...(params.subObjectId ? { subObjectId: params.subObjectId } : {}),
+    ...(params.subObjectType ?
+    { subObjectType: params.subObjectType } :
+    {}),
+    index_text_chunk: params.index_text_chunk,
+    embedding: params.embedding,
+    only_one_index_text_chunk: params.only_one_index_text_chunk,
+    index_text_chunk_length: params.index_text_chunk_length,
+    index_text_chunk_position: params.index_text_chunk_position,
+    index_text_chunk_overlap: params.index_text_chunk_overlap,
+    indexerId: params.indexerId,
+    createTime: params.createTime,
+    updateTime: params.updateTime
+  };
+}

package/dist/indexSchema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexSchema.js","sourceRoot":"","sources":["../src/indexSchema.ts"],"names":[],"mappings":"AAEA,MAAM,sBAAsB,GAAG,CAAC,CAAC;AAEjC,MAAM,UAAU,4BAA4B,CAAC,MAA8B;IACvE,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC;IACtD,MAAM,oBAAoB,GAAG,WAAW,CAAC,oBAAoB,CAAC;IAE9D,IAAI,WAAW,CAAC,YAAY,KAAK,sBAAsB,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CACX,oCAAoC,sBAAsB,SAAS,WAAW,CAAC,YAAY,EAAE,CAChG,CAAC;IACN,CAAC;IAED,IACI,WAAW,CAAC,oBAAoB,CAAC,gBAAgB;QACjD,WAAW,CAAC,oBAAoB,CAAC,OAAO,EAC1C,CAAC;QACC,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO;QACH,SAAS,EAAE,WAAW,CAAC,aAAa;QACpC,QAAQ,EAAE;YACN,KAAK,EAAE;gBACH,gBAAgB,EAAE,WAAW,CAAC,cAAc;gBAC5C,kBAAkB,EAAE,WAAW,CAAC,gBAAgB;aACnD;YACD,WAAW,EAAE,IAAI;SACpB;QACD,QAAQ,EAAE;YACN,UAAU,EAAE;gBACR,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC7B,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC7B,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC7B,cAAc,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACnC,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC/B,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAChC,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAClC,gBAAgB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBACrC,SAAS,EAAE;oBACP,IAAI,EAAE,YAAY;oBAClB,SAAS,EAAE,oBAAoB,CAAC,SAAS;oBACzC,UAAU,EAAE,oBAAoB,CAAC,SAAS;oBAC1C,IAAI,EAAE,oBAAoB,CAAC,IAAI;oBAC/B,GAAG,CAAC,oBAAoB,CAAC,gBAAgB;wBACrC,CAAC,CAAC;4BACI,iBAAiB,EACb,oBAAoB,CAAC,gBAAgB;yBAC5C;wBACH,CAAC,CAAC,EAAE,CAAC;oBACT,MAAM,EAAE;wBACJ,IAAI,EAAE,MAAM;wBACZ,MAAM,EAAE,OAAO;wBACf,UAAU,EAAE;4BACR,CAAC,EAAE,oBAAoB,CAAC,CAAC;4BACzB,eAAe,EACX,oBAAoB,CAAC,cAAc;4BACvC,SAAS,EAAE,oBAAoB,CAAC,QAAQ;4BACxC,GAAG,CAAC,oBAAoB,CAAC,OAAO;gCAC5B,CAAC,CAAC;oCACI,OAAO,EAAE;wCACL,IAAI,EACA,oBAAoB,CAAC,OAAO,CAAC,IAAI;wCACrC,UAAU,EAAE;4CACR,IAAI,EACA,oBAAoB,CAAC,OAAO;iDACvB,IAAI;4CACb,IAAI,EACA,oBAAoB,CAAC,OAAO;iDACvB,IAAI;yCAChB;qCACJ;iCACJ;gCACH,CAAC,CAAC,EAAE,CAAC;yBACZ;qBACJ;iBACJ;gBACD,yBAAyB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,uBAAuB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC5C,yBAAyB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9C,wBAAwB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC7C,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;gBAC9B,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE;gBACxD,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,kBAAkB,EAAE;aAC3D;SACJ;KACJ,CAAC;AACN,CAAC;AAuBD,MAAM,UAAU,0BAA0B,CACtC,MAA6B;IAE7B,OAAO;QACH,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,GAAG,CAAC,MAAM,CAAC,cAAc;YACrB,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE;YAC3C,CAAC,CAAC,EAAE,CAAC;QACT,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,MAAM,CAAC,aAAa;YACpB,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE;YACzC,CAAC,CAAC,EAAE,CAAC;QACT,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,yBAAyB,EAAE,MAAM,CAAC,yBAAyB;QAC3D,uBAAuB,EAAE,MAAM,CAAC,uBAAuB;QACvD,yBAAyB,EAAE,MAAM,CAAC,yBAAyB;QAC3D,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;QACzD,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;KAChC,CAAC;AACN,CAAC"}

package/dist/onRecordFoundRegistryRecord.d.ts

@@ -0,0 +1,7 @@
+import { onRecordFoundType } from "@magda/minion-framework/dist/MinionOptions.js";
+import { Chunker } from "./chunker.js";
+import EmbeddingApiClient from "@magda/typescript-common/dist/EmbeddingApiClient.js";
+import OpensearchApiClient from "@magda/typescript-common/dist/OpensearchApiClient.js";
+import SemanticIndexerOptions from "./semanticIndexerOptions.js";
+import Registry from "@magda/typescript-common/dist/registry/AuthorizedRegistryClient.js";
+export declare const onRecordFoundRegistryRecord: (options: SemanticIndexerOptions, chunker: Chunker, embeddingApiClient: EmbeddingApiClient, opensearchApiClient: OpensearchApiClient, registryReadonlyClient: Registry) => onRecordFoundType;

package/dist/onRecordFoundRegistryRecord.js

@@ -0,0 +1,47 @@
+import { indexEmbeddingText } from "./indexEmbeddingText.js";
+import { SkipError } from "./SkipError.js";
+// The onRecordFound function passed to minion sdk to handle registry record records
+export const onRecordFoundRegistryRecord = (options, chunker, embeddingApiClient, opensearchApiClient, registryReadonlyClient) => {
+  return async (record, _registry) => {
+    try {
+      if (!record.aspects ||
+      !options.aspects?.every((aspect) => aspect in record.aspects)) {
+        return;
+      }
+      let embeddingText;
+      try {
+        embeddingText = await options.createEmbeddingText({
+          record,
+          format: null,
+          filePath: null,
+          url: null,
+          readonlyRegistry: registryReadonlyClient
+        });
+        if (!embeddingText.text && !embeddingText.subObjects) {
+          throw new SkipError("User-provided createEmbeddingText function returned no text or subObjects");
+        }
+      }
+      catch (err) {
+        throw new SkipError(`Error in user-provided createEmbeddingText function: ${err.message}`);
+      }
+      await indexEmbeddingText({
+        options,
+        chunker,
+        embeddingApiClient,
+        opensearchApiClient,
+        embeddingText,
+        metadata: {
+          recordId: record.id,
+          fileFormat: record.aspects["dataset-format"]?.format
+        }
+      });
+    }
+    catch (err) {
+      if (err instanceof SkipError) {
+        console.warn(`Skipping record ${record.id} because:`, err.message);
+        return;
+      }
+      throw err;
+    }
+  };
+};

package/dist/onRecordFoundRegistryRecord.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"onRecordFoundRegistryRecord.js","sourceRoot":"","sources":["../src/onRecordFoundRegistryRecord.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C,oFAAoF;AACpF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CACvC,OAA+B,EAC/B,OAAgB,EAChB,kBAAsC,EACtC,mBAAwC,EACxC,sBAAgC,EACf,EAAE;IACnB,OAAO,KAAK,EAAE,MAAc,EAAE,SAAS,EAAE,EAAE;QACvC,IAAI,CAAC;YACD,IACI,CAAC,MAAM,CAAC,OAAO;gBACf,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,EAC/D,CAAC;gBACC,OAAO;YACX,CAAC;YAED,IAAI,aAAa,CAAC;YAClB,IAAI,CAAC;gBACD,aAAa,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC;oBAC9C,MAAM;oBACN,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,IAAI;oBACd,GAAG,EAAE,IAAI;oBACT,gBAAgB,EAAE,sBAAsB;iBAC3C,CAAC,CAAC;gBAEH,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;oBACnD,MAAM,IAAI,SAAS,CACf,2EAA2E,CAC9E,CAAC;gBACN,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,IAAI,SAAS,CACf,wDACK,GAAa,CAAC,OACnB,EAAE,CACL,CAAC;YACN,CAAC;YAED,MAAM,kBAAkB,CAAC;gBACrB,OAAO;gBACP,OAAO;gBACP,kBAAkB;gBAClB,mBAAmB;gBACnB,aAAa;gBACb,QAAQ,EAAE;oBACN,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,MAAM;iBACvD;aACJ,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;gBAC3B,OAAO,CAAC,IAAI,CACR,mBAAmB,MAAM,CAAC,EAAE,WAAW,EACvC,GAAG,CAAC,OAAO,CACd,CAAC;gBACF,OAAO;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC,CAAC;AACN,CAAC,CAAC"}

package/dist/onRecordFoundStorageObject.d.ts

@@ -0,0 +1,9 @@
+import { onRecordFoundType } from "@magda/minion-framework/dist/MinionOptions.js";
+import { Chunker } from "./chunker.js";
+import EmbeddingApiClient from "@magda/typescript-common/dist/EmbeddingApiClient.js";
+import OpensearchApiClient from "@magda/typescript-common/dist/OpensearchApiClient.js";
+import SemanticIndexerOptions from "./semanticIndexerOptions.js";
+import Registry from "@magda/typescript-common/dist/registry/AuthorizedRegistryClient.js";
+import { MinioClient } from "./MinioClient.js";
+export declare const onRecordFoundStorageObject: (options: SemanticIndexerOptions, chunker: Chunker, embeddingApiClient: EmbeddingApiClient, opensearchApiClient: OpensearchApiClient, minioClient: MinioClient, registryReadonlyClient: Registry) => onRecordFoundType;
+export declare function getParentRecordId(distributionId: string, registry: Registry): Promise<string | null>;

package/dist/onRecordFoundStorageObject.js

@@ -0,0 +1,145 @@
+import { indexEmbeddingText } from "./indexEmbeddingText.js";
+import retry from "@magda/typescript-common/dist/retry.js";
+import fetch from "node-fetch";
+import * as fs from "fs";
+import { tmpdir } from "os";
+import { join } from "path";
+import { v4 as uuidv4 } from "uuid";
+import { SkipError } from "./SkipError.js";
+import urijs from "urijs";
+import { pipeline } from "stream/promises";
+import ServerError from "@magda/typescript-common/dist/ServerError.js";
+import { deleteTempFile } from "./helpers.js";
+// The onRecordFound function passed to minion sdk to handle storage object records
+export const onRecordFoundStorageObject = (options, chunker, embeddingApiClient, opensearchApiClient, minioClient, registryReadonlyClient) => {
+  return async (dist, registry) => {
+    try {
+      const datasetFormat = dist.aspects?.["dataset-format"]?.format;
+      const dcatDist = dist.aspects?.["dcat-distribution-strings"] || {};
+      const { format: dcatFormat, downloadURL, accessURL } = dcatDist;
+      const fileDownloadURL = downloadURL || accessURL;
+      let format = datasetFormat || dcatFormat;
+      if (!format && fileDownloadURL) {
+        format = new urijs(fileDownloadURL).suffix().toUpperCase();
+      }
+      // filler record
+      if (!format ||
+      !fileDownloadURL ||
+      !options.formatTypes?.some((f) => format.toLowerCase().includes(f.toLowerCase()))) {
+        return;
+      }
+      let embeddingText;
+      let filePath = null;
+      const parentRecordId = await getParentRecordId(dist.id, registryReadonlyClient);
+      try {
+        try {
+          if (options.autoDownloadFile === undefined ||
+          options.autoDownloadFile) {
+            filePath = await downloadFileWithRetry(fileDownloadURL, minioClient);
+          }
+        }
+        catch (err) {
+          throw new SkipError(`Error in downloading file: ${err.message}`);
+        }
+        try {
+          embeddingText = await options.createEmbeddingText({
+            record: dist,
+            format: format,
+            filePath,
+            url: fileDownloadURL,
+            readonlyRegistry: registryReadonlyClient
+          });
+          if (!embeddingText.text && !embeddingText.subObjects) {
+            throw new SkipError("User-provided createEmbeddingText function returned no text or subObjects");
+          }
+        }
+        catch (err) {
+          throw new SkipError(`Error in user-provided createEmbeddingText function: ${err.message}`);
+        }
+      } finally
+      {
+        if (filePath) {
+          await deleteTempFile(filePath);
+        }
+      }
+      await indexEmbeddingText({
+        options,
+        chunker,
+        embeddingApiClient,
+        opensearchApiClient,
+        embeddingText,
+        metadata: {
+          recordId: dist.id,
+          parentRecordId: parentRecordId,
+          aspectId: dist.aspects["dataset-format"]?.id,
+          fileFormat: format
+        }
+      });
+    }
+    catch (err) {
+      if (err instanceof SkipError) {
+        console.warn(`Skipping distribution ${dist.id} because:`, err.message);
+        return;
+      }
+      throw err;
+    }
+  };
+};
+async function downloadFileWithRetry(url, minioClient) {
+  return retry(() => downloadFile(url, minioClient), 1, 5, (err, retries) => {});
+}
+async function downloadFile(url, minioClient) {
+  const uri = urijs(url);
+  if (uri.protocol() === "magda" && uri.hostname() === "storage-api") {
+    try {
+      return await minioClient.downloadFile(url);
+    }
+    catch (err) {
+      throw new SkipError(`Failed to download file from Minio: ${err.message}`);
+    }
+  }
+  let response;
+  try {
+    response = await fetch(url);
+  }
+  catch (err) {
+    throw new SkipError(`Failed to download file because network error`);
+  }
+  if (!response.ok) {
+    throw new SkipError(`Failed to download file because HTTP error ${response.status}`);
+  }
+  if (!response.body) {
+    throw new SkipError("No response body to write to file");
+  }
+  const tempDir = tmpdir();
+  const tempFileName = `${uuidv4()}`;
+  const suffix = new urijs(url).suffix();
+  const tempFilePath = join(tempDir, `${tempFileName}.${suffix}`);
+  try {
+    const writeStream = fs.createWriteStream(tempFilePath);
+    await pipeline(response.body, writeStream);
+    return tempFilePath;
+  }
+  catch (err) {
+    await deleteTempFile(tempFilePath);
+    throw new SkipError(`Failed to write file`);
+  }
+}
+export async function getParentRecordId(distributionId, registry) {
+  try {
+    const result = await registry.getRecords(["dataset-distributions"], undefined, undefined, true, undefined, ["dataset-distributions.distributions:<|" + distributionId]);
+    if (result instanceof ServerError) {
+      console.error(`Failed to get parent record id: ${result.message}`);
+      return null;
+    }
+    if (!("records" in result)) {
+      console.error(`Failed to get parent record id`);
+      return null;
+    }
+    return result.records[0]?.id || null;
+  }
+  catch (e) {
+    console.error(`Unexpected error when getting parent record id`, e);
+    return null;
+  }
+}

package/dist/onRecordFoundStorageObject.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"onRecordFoundStorageObject.js","sourceRoot":"","sources":["../src/onRecordFoundStorageObject.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAG7D,OAAO,KAAK,MAAM,sCAAsC,CAAC;AACzD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,WAAW,MAAM,4CAA4C,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,mFAAmF;AACnF,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACtC,OAA+B,EAC/B,OAAgB,EAChB,kBAAsC,EACtC,mBAAwC,EACxC,WAAwB,EACxB,sBAAgC,EACf,EAAE;IACnB,OAAO,KAAK,EAAE,IAAY,EAAE,QAAQ,EAAE,EAAE;QACpC,IAAI,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,2BAA2B,CAAC,IAAI,EAAE,CAAC;YACnE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,QAAQ,CAAC;YAChE,MAAM,eAAe,GAAG,WAAW,IAAI,SAAS,CAAC;YACjD,IAAI,MAAM,GAAG,aAAa,IAAI,UAAU,CAAC;YACzC,IAAI,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;gBAC7B,MAAM,GAAG,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC;YAC/D,CAAC;YAED,gBAAgB;YAChB,IACI,CAAC,MAAM;gBACP,CAAC,eAAe;gBAChB,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7B,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CACjD,EACH,CAAC;gBACC,OAAO;YACX,CAAC;YAED,IAAI,aAA4B,CAAC;YACjC,IAAI,QAAQ,GAAkB,IAAI,CAAC;YAEnC,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAC1C,IAAI,CAAC,EAAE,EACP,sBAAsB,CACzB,CAAC;YACF,IAAI,CAAC;gBACD,IAAI,CAAC;oBACD,IACI,OAAO,CAAC,gBAAgB,KAAK,SAAS;wBACtC,OAAO,CAAC,gBAAgB,EAC1B,CAAC;wBACC,QAAQ,GAAG,MAAM,qBAAqB,CAClC,eAAe,EACf,WAAW,CACd,CAAC;oBACN,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,MAAM,IAAI,SAAS,CACf,8BAA+B,GAAa,CAAC,OAAO,EAAE,CACzD,CAAC;gBACN,CAAC;gBACD,IAAI,CAAC;oBACD,aAAa,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC;wBAC9C,MAAM,EAAE,IAAI;wBACZ,MAAM,EAAE,MAAM;wBACd,QAAQ;wBACR,GAAG,EAAE,eAAe;wBACpB,gBAAgB,EAAE,sBAAsB;qBAC3C,CAAC,CAAC;oBAEH,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;wBACnD,MAAM,IAAI,SAAS,CACf,2EAA2E,CAC9E,CAAC;oBACN,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,MAAM,IAAI,SAAS,CACf,wDACK,GAAa,CAAC,OACnB,EAAE,CACL,CAAC;gBACN,CAAC;YACL,CAAC;oBAAS,CAAC;gBACP,IAAI,QAAQ,EAAE,CAAC;oBACX,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;YAED,MAAM,kBAAkB,CAAC;gBACrB,OAAO;gBACP,OAAO;gBACP,kBAAkB;gBAClB,mBAAmB;gBACnB,aAAa;gBACb,QAAQ,EAAE;oBACN,QAAQ,EAAE,IAAI,CAAC,EAAE;oBACjB,cAAc,EAAE,cAAc;oBAC9B,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,EAAE;oBAC5C,UAAU,EAAE,MAAM;iBACrB;aACJ,CAAC,CAAC;QACP,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;gBAC3B,OAAO,CAAC,IAAI,CACR,yBAAyB,IAAI,CAAC,EAAE,WAAW,EAC3C,GAAG,CAAC,OAAO,CACd,CAAC;gBACF,OAAO;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,KAAK,UAAU,qBAAqB,CAChC,GAAW,EACX,WAAwB;IAExB,OAAO,KAAK,CACR,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,WAAW,CAAC,EACpC,CAAC,EACD,CAAC,EACD,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,GAAE,CAAC,CACvB,CAAC;AACN,CAAC;AAED,KAAK,UAAU,YAAY,CACvB,GAAW,EACX,WAAwB;IAExB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,EAAE,KAAK,aAAa,EAAE,CAAC;QACjE,IAAI,CAAC;YACD,OAAO,MAAM,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,SAAS,CACf,uCAAwC,GAAa,CAAC,OAAO,EAAE,CAClE,CAAC;QACN,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACD,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,SAAS,CACf,8CAA8C,QAAQ,CAAC,MAAM,EAAE,CAClE,CAAC;IACN,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;IACzB,MAAM,YAAY,GAAG,GAAG,MAAM,EAAE,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,IAAI,MAAM,EAAE,CAAC,CAAC;IAEhE,IAAI,CAAC;QACD,MAAM,WAAW,GAAG,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC3C,OAAO,YAAY,CAAC;IACxB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,cAAc,CAAC,YAAY,CAAC,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,sBAAsB,CAAC,CAAC;IAChD,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACnC,cAAsB,EACtB,QAAkB;IAElB,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,UAAU,CACpC,CAAC,uBAAuB,CAAC,EACzB,SAAS,EACT,SAAS,EACT,IAAI,EACJ,SAAS,EACT,CAAC,wCAAwC,GAAG,cAAc,CAAC,CAC9D,CAAC;QAEF,IAAI,MAAM,YAAY,WAAW,EAAE,CAAC;YAChC,OAAO,CAAC,KAAK,CAAC,mCAAmC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAChD,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC;IACzC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC"}

package/dist/semanticIndexer.d.ts

@@ -0,0 +1,2 @@
+import SemanticIndexerOptions from "./semanticIndexerOptions.js";
+export default function semanticIndexer(userConfig: SemanticIndexerOptions): Promise<void>;

package/dist/semanticIndexer.js

@@ -0,0 +1,86 @@
+import minion from "@magda/minion-framework/dist/index.js";
+import { Chunker, UserDefinedChunkStrategy, RecursiveChunkStrategy } from "./chunker.js";
+import { onRecordFoundRegistryRecord } from "./onRecordFoundRegistryRecord.js";
+import { onRecordFoundStorageObject } from "./onRecordFoundStorageObject.js";
+import { createSemanticIndexerMapping } from "./indexSchema.js";
+import EmbeddingApiClient from "@magda/typescript-common/dist/EmbeddingApiClient.js";
+import OpensearchApiClient from "@magda/typescript-common/dist/OpensearchApiClient.js";
+import { validateSemanticIndexerOptions } from "./semanticIndexerOptions.js";
+import retry from "@magda/typescript-common/dist/retry.js";
+import { MinioClient } from "./MinioClient.js";
+import { MAGDA_SYSTEM_ID } from "@magda/typescript-common/dist/registry/TenantConsts.js";
+import Registry from "@magda/typescript-common/dist/registry/AuthorizedRegistryClient.js";
+// Main function for semantic indexer
+export default async function semanticIndexer(userConfig) {
+  try {
+    validateSemanticIndexerOptions(userConfig);
+    const semanticIndexerConfig = userConfig.argv.semanticIndexerConfig;
+    const opensearchApiClient = await retry(() => OpensearchApiClient.getOpensearchApiClient({
+      url: userConfig.argv.opensearchApiURL
+    }), 5, 5, (e, left) => console.error(`Opensearch connection failed, remaining retries: ${left}, error:`, e.message));
+    const embeddingApiClient = await retry(() => Promise.resolve(new EmbeddingApiClient({
+      baseApiUrl: userConfig.argv.embeddingApiURL
+    })), 5, 5, (e, left) => console.error(`Embedding API connection failed, remaining retries: ${left}, error:`, e.message));
+    const registryReadonlyClient = new Registry({
+      baseUrl: userConfig.argv.registryReadonlyURL,
+      jwtSecret: userConfig.argv.jwtSecret,
+      userId: userConfig.argv.userId,
+      maxRetries: 3,
+      tenantId: MAGDA_SYSTEM_ID
+    });
+    if (!(await opensearchApiClient.indexExists(semanticIndexerConfig.fullIndexName))) {
+      const indexDefinition = createSemanticIndexerMapping(userConfig);
+      await opensearchApiClient.createIndex(indexDefinition);
+    }
+    const chunkStrategy = userConfig.chunkStrategy ?
+    new UserDefinedChunkStrategy(userConfig.chunkStrategy) :
+    new RecursiveChunkStrategy(semanticIndexerConfig.chunkSizeLimit ||
+    userConfig.chunkSizeLimit, semanticIndexerConfig.overlap || userConfig.overlap);
+    const chunker = new Chunker(chunkStrategy);
+    let onRecordFound;
+    let minionOptions;
+    if (userConfig.itemType === "registryRecord") {
+      onRecordFound = onRecordFoundRegistryRecord(userConfig, chunker, embeddingApiClient, opensearchApiClient, registryReadonlyClient);
+      minionOptions = {
+        argv: userConfig.argv,
+        id: userConfig.id,
+        aspects: userConfig.aspects || [],
+        optionalAspects: userConfig.optionalAspects || [],
+        writeAspectDefs: [],
+        async: true,
+        dereference: true,
+        includeEvents: false,
+        includeRecords: true,
+        onRecordFound: onRecordFound
+      };
+    } else
+    if (userConfig.itemType === "storageObject") {
+      const minioClient = new MinioClient(userConfig.argv.minioConfig, userConfig.argv.minioAccessKey, userConfig.argv.minioSecretKey);
+      onRecordFound = onRecordFoundStorageObject(userConfig, chunker, embeddingApiClient, opensearchApiClient, minioClient, registryReadonlyClient);
+      minionOptions = {
+        argv: userConfig.argv,
+        id: userConfig.id,
+        aspects: ["dcat-distribution-strings", "dataset-format"],
+        optionalAspects: [],
+        writeAspectDefs: [],
+        async: true,
+        dereference: false,
+        includeEvents: false,
+        includeRecords: true,
+        onRecordFound: onRecordFound,
+        maxRetries: 3
+      };
+    } else
+    {
+      throw new Error("Invalid itemType");
+    }
+    minion(minionOptions).catch((e) => {
+      console.error("Minion execution error: " + e.message, e);
+      process.exit(1);
+    });
+  }
+  catch (e) {
+    console.error("semanticIndexer initialization error: " + e.message, e);
+    process.exit(1);
+  }
+}

package/dist/semanticIndexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semanticIndexer.js","sourceRoot":"","sources":["../src/semanticIndexer.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,qCAAqC,CAAC;AACzD,OAAO,EACH,OAAO,EACP,wBAAwB,EACxB,sBAAsB,EACzB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,2BAA2B,EAAE,MAAM,kCAAkC,CAAC;AAC/E,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAC7E,OAAO,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,kBAAkB,MAAM,mDAAmD,CAAC;AACnF,OAAO,mBAAmB,MAAM,oDAAoD,CAAC;AACrF,OAA+B,EAC3B,8BAA8B,EACjC,MAAM,6BAA6B,CAAC;AAIrC,OAAO,KAAK,MAAM,sCAAsC,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,sDAAsD,CAAC;AACvF,OAAO,QAAQ,MAAM,kEAAkE,CAAC;AAExF,qCAAqC;AACrC,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,eAAe,CACzC,UAAkC;IAElC,IAAI,CAAC;QACD,8BAA8B,CAAC,UAAU,CAAC,CAAC;QAC3C,MAAM,qBAAqB,GAAG,UAAU,CAAC,IAAI,CAAC,qBAAqB,CAAC;QACpE,MAAM,mBAAmB,GAAG,MAAM,KAAK,CACnC,GAAG,EAAE,CACD,mBAAmB,CAAC,sBAAsB,CAAC;YACvC,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,gBAAgB;SACxC,CAAC,EACN,CAAC,EACD,CAAC,EACD,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CACR,OAAO,CAAC,KAAK,CACT,oDAAoD,IAAI,UAAU,EAClE,CAAC,CAAC,OAAO,CACZ,CACR,CAAC;QAEF,MAAM,kBAAkB,GAAG,MAAM,KAAK,CAClC,GAAG,EAAE,CACD,OAAO,CAAC,OAAO,CACX,IAAI,kBAAkB,CAAC;YACnB,UAAU,EAAE,UAAU,CAAC,IAAI,CAAC,eAAe;SAC9C,CAAC,CACL,EACL,CAAC,EACD,CAAC,EACD,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CACR,OAAO,CAAC,KAAK,CACT,uDAAuD,IAAI,UAAU,EACrE,CAAC,CAAC,OAAO,CACZ,CACR,CAAC;QAEF,MAAM,sBAAsB,GAAG,IAAI,QAAQ,CAAC;YACxC,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,mBAAmB;YAC5C,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,SAAS;YACpC,MAAM,EAAE,UAAU,CAAC,IAAI,CAAC,MAAM;YAC9B,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,eAAe;SAC5B,CAAC,CAAC;QAEH,IACI,CAAC,CAAC,MAAM,mBAAmB,CAAC,WAAW,CACnC,qBAAqB,CAAC,aAAa,CACtC,CAAC,EACJ,CAAC;YACC,MAAM,eAAe,GAAG,4BAA4B,CAAC,UAAU,CAAC,CAAC;YACjE,MAAM,mBAAmB,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,UAAU,CAAC,aAAa;YAC1C,CAAC,CAAC,IAAI,wBAAwB,CAAC,UAAU,CAAC,aAAa,CAAC;YACxD,CAAC,CAAC,IAAI,sBAAsB,CACtB,qBAAqB,CAAC,cAAc;gBAChC,UAAU,CAAC,cAAc,EAC7B,qBAAqB,CAAC,OAAO,IAAI,UAAU,CAAC,OAAO,CACtD,CAAC;QAER,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;QAE3C,IAAI,aAAgC,CAAC;QACrC,IAAI,aAA4B,CAAC;QAEjC,IAAI,UAAU,CAAC,QAAQ,KAAK,gBAAgB,EAAE,CAAC;YAC3C,aAAa,GAAG,2BAA2B,CACvC,UAAU,EACV,OAAO,EACP,kBAAkB,EAClB,mBAAmB,EACnB,sBAAsB,CACzB,CAAC;YACF,aAAa,GAAG;gBACZ,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,EAAE,EAAE,UAAU,CAAC,EAAE;gBACjB,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,EAAE;gBACjC,eAAe,EAAE,UAAU,CAAC,eAAe,IAAI,EAAE;gBACjD,eAAe,EAAE,EAAE;gBACnB,KAAK,EAAE,IAAI;gBACX,WAAW,EAAE,IAAI;gBACjB,aAAa,EAAE,KAAK;gBACpB,cAAc,EAAE,IAAI;gBACpB,aAAa,EAAE,aAAa;aAC/B,CAAC;QACN,CAAC;aAAM,IAAI,UAAU,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;YACjD,MAAM,WAAW,GAAG,IAAI,WAAW,CAC/B,UAAU,CAAC,IAAI,CAAC,WAAW,EAC3B,UAAU,CAAC,IAAI,CAAC,cAAc,EAC9B,UAAU,CAAC,IAAI,CAAC,cAAc,CACjC,CAAC;YACF,aAAa,GAAG,0BAA0B,CACtC,UAAU,EACV,OAAO,EACP,kBAAkB,EAClB,mBAAmB,EACnB,WAAW,EACX,sBAAsB,CACzB,CAAC;YACF,aAAa,GAAG;gBACZ,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,EAAE,EAAE,UAAU,CAAC,EAAE;gBACjB,OAAO,EAAE,CAAC,2BAA2B,EAAE,gBAAgB,CAAC;gBACxD,eAAe,EAAE,EAAE;gBACnB,eAAe,EAAE,EAAE;gBACnB,KAAK,EAAE,IAAI;gBACX,WAAW,EAAE,KAAK;gBAClB,aAAa,EAAE,KAAK;gBACpB,cAAc,EAAE,IAAI;gBACpB,aAAa,EAAE,aAAa;gBAC5B,UAAU,EAAE,CAAC;aAChB,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,CAAQ,EAAE,EAAE;YACrC,OAAO,CAAC,KAAK,CAAC,0BAA0B,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC,CAAC,CAAC;IACP,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CACT,wCAAwC,GAAI,CAAW,CAAC,OAAO,EAC/D,CAAC,CACJ,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;AACL,CAAC"}

package/dist/semanticIndexerOptions.d.ts

@@ -0,0 +1,19 @@
+import { ItemType } from "./indexSchema.js";
+import { CreateEmbeddingText } from "./createEmbeddingText.js";
+import { SemanticIndexerArguments } from "./commonYargs.js";
+import { ChunkStrategyType } from "./chunker.js";
+export default interface SemanticIndexerOptions {
+  argv: SemanticIndexerArguments;
+  id: string;
+  itemType: ItemType;
+  aspects?: string[];
+  optionalAspects?: string[];
+  formatTypes?: string[];
+  createEmbeddingText: CreateEmbeddingText;
+  chunkStrategy?: ChunkStrategyType;
+  chunkSizeLimit?: number;
+  overlap?: number;
+  autoDownloadFile?: boolean;
+  timeout?: string;
+}
+export declare function validateSemanticIndexerOptions(options: SemanticIndexerOptions): void;

package/dist/semanticIndexerOptions.js

@@ -0,0 +1,26 @@
+export function validateSemanticIndexerOptions(options) {
+  if (options.chunkSizeLimit !== undefined && (
+  !Number.isInteger(options.chunkSizeLimit) ||
+  options.chunkSizeLimit <= 0)) {
+    throw new Error("'chunkSizeLimit' must be a positive integer");
+  }
+  if (options.overlap !== undefined && (
+  !Number.isInteger(options.overlap) || options.overlap < 0)) {
+    throw new Error("'overlap' must be a non-negative integer");
+  }
+  if (options.chunkSizeLimit &&
+  options.overlap &&
+  options.chunkSizeLimit <= options.overlap * 2) {
+    throw new Error("'overlap' must be less than half of 'chunkSizeLimit'");
+  }
+  if (options.itemType === "registryRecord") {
+    if (!options.aspects || options.aspects.length === 0) {
+      throw new Error("'aspects' is required");
+    }
+  }
+  if (options.itemType === "storageObject") {
+    if (!options.formatTypes || options.formatTypes.length === 0) {
+      throw new Error("'formatTypes' is required");
+    }
+  }
+}

package/dist/semanticIndexerOptions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semanticIndexerOptions.js","sourceRoot":"","sources":["../src/semanticIndexerOptions.ts"],"names":[],"mappings":"AAqBA,MAAM,UAAU,8BAA8B,CAC1C,OAA+B;IAE/B,IACI,OAAO,CAAC,cAAc,KAAK,SAAS;QACpC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,cAAc,CAAC;YACtC,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,EAClC,CAAC;QACC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACnE,CAAC;IACD,IACI,OAAO,CAAC,OAAO,KAAK,SAAS;QAC7B,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC,EAC7D,CAAC;QACC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IACD,IACI,OAAO,CAAC,cAAc;QACtB,OAAO,CAAC,OAAO;QACf,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC,EAC/C,CAAC;QACC,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,gBAAgB,EAAE,CAAC;QACxC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC7C,CAAC;IACL,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,eAAe,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QACjD,CAAC;IACL,CAAC;AACL,CAAC"}

package/dist/test/BaseSemanticIndexerTest.d.ts

@@ -0,0 +1,41 @@
+import { SinonStub, SinonFakeTimers } from "sinon";
+import SemanticIndexerOptions from "../semanticIndexerOptions.js";
+export declare class BaseSemanticIndexerTest {
+  chunker: any;
+  embeddingApiClient: any;
+  opensearchApiClient: any;
+  minioClient: any;
+  registry: any;
+  createEmbeddingTextStub: SinonStub;
+  consoleLogStub: SinonStub;
+  consoleWarnStub: SinonStub;
+  consoleErrorStub: SinonStub;
+  clock: SinonFakeTimers;
+  DEFAULT_FAKE_TIME: Date;
+  DEFAULT_PARENT_RECORD_ID: string;
+  DEFAULT_CREATE_EMBEDDING_TEXT_RESULT: {
+    text: string;
+  };
+  userConfig: any;
+  constructor({ createEmbeddingTextResult, fakeTime, overridesConfig, suppressConsoleLogs
+
+
+
+
+  }?: {createEmbeddingTextResult?: any;fakeTime?: Date;overridesConfig?: Partial<SemanticIndexerOptions>;suppressConsoleLogs?: boolean;});
+  cleanup(): void;
+  updateUserConfig(overrides?: Partial<SemanticIndexerOptions>): any;
+  getCurrentTimeString(): string;
+  expectCalledWith(stub: SinonStub, callIndex: number, ...expectedArgs: any[]): void;
+  expectSuccessCalls(options?: {
+    createEmbeddingTextCallCount?: number;
+    chunkCallCount?: number;
+    embeddingApiCallCount?: number;
+    bulkIndexCallCount?: number;
+    deleteByQueryCallCount?: number;
+  }): void;
+  expectIndexedDoc(expectedDoc: any, callIndex?: number): void;
+  expectIndexedDocs(expectedDocs: any[], callIndex?: number): void;
+  getIndexedDocs(callIndex?: number): any[];
+}
+export declare function createFakeSemanticIndexerConfig(overrideConfig?: Partial<SemanticIndexerOptions>): SemanticIndexerOptions;