@magda/semantic-indexer-framework 5.2.0

package/dist/MinioClient.d.ts

@@ -0,0 +1,7 @@
+import { MinioConfig } from "./commonYargs.js";
+export declare class MinioClient {
+  private client;
+  private config;
+  constructor(config: MinioConfig, accessKey: string, secretKey: string);
+  downloadFile(url: string): Promise<string>;
+}

package/dist/MinioClient.js

@@ -0,0 +1,37 @@
+import * as Minio from "minio";
+import urijs from "urijs";
+import { tmpdir } from "os";
+import { v4 as uuidv4 } from "uuid";
+import { join } from "path";
+import { deleteTempFile } from "./helpers.js";
+export class MinioClient {
+  client;
+  config;
+  constructor(config, accessKey, secretKey) {
+    this.client = new Minio.Client({
+      endPoint: config.endPoint,
+      port: config.port,
+      accessKey: accessKey,
+      secretKey: secretKey,
+      useSSL: config.useSSL,
+      region: config.region
+    });
+    this.config = config;
+  }
+  async downloadFile(url) {
+    const uri = urijs(url);
+    const [datasetId, distributionId, fileName] = uri.segmentCoded();
+    const objectName = `${datasetId}/${distributionId}/${fileName}`;
+    const tempDir = tmpdir();
+    const tempFileName = `${uuidv4()}`;
+    const tempFilePath = join(tempDir, tempFileName);
+    try {
+      await this.client.fGetObject(this.config.defaultDatasetBucket, objectName, tempFilePath);
+      return tempFilePath;
+    }
+    catch (err) {
+      await deleteTempFile(tempFilePath);
+      throw err;
+    }
+  }
+}

package/dist/MinioClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MinioClient.js","sourceRoot":"","sources":["../src/MinioClient.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAC/B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,OAAO,WAAW;IACZ,MAAM,CAAe;IACrB,MAAM,CAAc;IAE5B,YAAY,MAAmB,EAAE,SAAiB,EAAE,SAAiB;QACjE,IAAI,CAAC,MAAM,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,SAAS,EAAE,SAAS;YACpB,SAAS,EAAE,SAAS;YACpB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,MAAM,CAAC,SAAS,EAAE,cAAc,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;QAEjE,MAAM,UAAU,GAAG,GAAG,SAAS,IAAI,cAAc,IAAI,QAAQ,EAAE,CAAC;QAEhE,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC;QACzB,MAAM,YAAY,GAAG,GAAG,MAAM,EAAE,EAAE,CAAC;QACnC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QAEjD,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CACxB,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAChC,UAAU,EACV,YAAY,CACf,CAAC;YACF,OAAO,YAAY,CAAC;QACxB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,cAAc,CAAC,YAAY,CAAC,CAAC;YACnC,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/dist/SkipError.d.ts

@@ -0,0 +1 @@
+export declare class SkipError extends Error {}

package/dist/SkipError.js

@@ -0,0 +1 @@
+export class SkipError extends Error {}

package/dist/SkipError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SkipError.js","sourceRoot":"","sources":["../src/SkipError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,SAAU,SAAQ,KAAK;CAAG"}

package/dist/chunker.d.ts

@@ -0,0 +1,32 @@
+export interface ChunkResult {
+  text: string;
+  position: number;
+  length: number;
+  overlap: number;
+}
+export type ChunkStrategyType = (text: string) => Promise<ChunkResult[]>;
+export interface ChunkStrategy {
+  chunk: ChunkStrategyType;
+}
+export declare class Chunker {
+  private strategy;
+  constructor(strategy: ChunkStrategy);
+  chunk(text: string): Promise<ChunkResult[]>;
+}
+export declare class UserDefinedChunkStrategy implements ChunkStrategy {
+  private chunkFunction;
+  constructor(chunkFunction: ChunkStrategyType);
+  chunk(text: string): Promise<ChunkResult[]>;
+}
+export declare class RecursiveChunkStrategy {
+  private chunkSize;
+  private overlap;
+  constructor(chunkSize: number, overlap: number);
+  chunk(text: string): Promise<ChunkResult[]>;
+}
+export declare class FixedLengthChunkStrategy implements ChunkStrategy {
+  private chunkSize;
+  private overlap;
+  constructor(chunkSize: number, overlap: number);
+  chunk(text: string): Promise<ChunkResult[]>;
+}

package/dist/chunker.js

@@ -0,0 +1,84 @@
+import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
+export class Chunker {
+  strategy;
+  constructor(strategy) {
+    this.strategy = strategy;
+  }
+  async chunk(text) {
+    return this.strategy.chunk(text);
+  }
+}
+export class UserDefinedChunkStrategy {
+  chunkFunction;
+  constructor(chunkFunction) {
+    this.chunkFunction = chunkFunction;
+  }
+  async chunk(text) {
+    return this.chunkFunction(text);
+  }
+}
+export class RecursiveChunkStrategy {
+  chunkSize;
+  overlap;
+  constructor(chunkSize, overlap) {
+    if (overlap >= chunkSize)
+    throw new Error("overlap must be < chunkSize");
+    this.chunkSize = chunkSize * 4;
+    this.overlap = overlap * 4;
+  }
+  async chunk(text) {
+    if (!text) {
+      return [];
+    }
+    if (text.length <= this.chunkSize) {
+      return [
+      {
+        text: text,
+        position: 0,
+        length: text.length,
+        overlap: 0
+      }];
+
+    }
+    const splitter = new RecursiveCharacterTextSplitter({
+      chunkSize: this.chunkSize,
+      chunkOverlap: this.overlap,
+      keepSeparator: true,
+      stripWhitespace: false
+    });
+    const chunks = await splitter.splitTextWithMetadata(text);
+    for (let i = 1; i < chunks.length; i++) {
+      if (chunks[i].overlap === 0 && chunks[i].text.trim() === "") {
+        chunks[i - 1].text = chunks[i - 1].text + chunks[i].text;
+        chunks[i - 1].length = chunks[i - 1].length + chunks[i].length;
+        chunks.splice(i, 1);
+        i--;
+      }
+    }
+    return chunks;
+  }
+}
+export class FixedLengthChunkStrategy {
+  chunkSize;
+  overlap;
+  constructor(chunkSize, overlap) {
+    this.chunkSize = chunkSize;
+    this.overlap = overlap;
+    if (this.overlap >= this.chunkSize / 2) {
+      throw new Error("overlap must be less than half of chunk size");
+    }
+  }
+  async chunk(text) {
+    const chunks = [];
+    for (let i = 0; i < text.length - this.overlap; i += this.chunkSize - this.overlap) {
+      const chunkText = text.slice(i, i + this.chunkSize);
+      chunks.push({
+        text: chunkText,
+        position: i,
+        length: chunkText.length,
+        overlap: this.overlap
+      });
+    }
+    return chunks;
+  }
+}

package/dist/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AAe1E,MAAM,OAAO,OAAO;IACR,QAAQ,CAAgB;IAChC,YAAY,QAAuB;QAC/B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;CACJ;AAED,MAAM,OAAO,wBAAwB;IACzB,aAAa,CAAoB;IAEzC,YAAY,aAAgC;QACxC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IACvB,SAAS,CAAC;IACV,OAAO,CAAC;IAEhB,YAAY,SAAiB,EAAE,OAAe;QAC1C,IAAI,OAAO,IAAI,SAAS;YACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,EAAE,CAAC;QACd,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAChC,OAAO;gBACH;oBACI,IAAI,EAAE,IAAI;oBACV,QAAQ,EAAE,CAAC;oBACX,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,OAAO,EAAE,CAAC;iBACb;aACJ,CAAC;QACN,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,8BAA8B,CAAC;YAChD,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,YAAY,EAAE,IAAI,CAAC,OAAO;YAC1B,aAAa,EAAE,IAAI;YACnB,eAAe,EAAE,KAAK;SACzB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC1D,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACzD,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACpB,CAAC,EAAE,CAAC;YACR,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AAED,MAAM,OAAO,wBAAwB;IACzB,SAAS,CAAS;IAClB,OAAO,CAAS;IAExB,YAAY,SAAiB,EAAE,OAAe;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACpE,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACpB,MAAM,MAAM,GAAkB,EAAE,CAAC;QACjC,KACI,IAAI,CAAC,GAAG,CAAC,EACT,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,EAC9B,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,EACpC,CAAC;YACC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,OAAO,EAAE,IAAI,CAAC,OAAO;aACxB,CAAC,CAAC;QACP,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ"}

package/dist/commonYargs.d.ts

@@ -0,0 +1,43 @@
+import { MinionArguments } from "@magda/minion-framework/dist/commonYargs.js";
+export interface SemanticIndexerConfig {
+  numberOfShards: number;
+  numberOfReplicas: number;
+  indexName: string;
+  indexVersion: number;
+  chunkSizeLimit: number;
+  overlap: number;
+  bulkEmbeddingsSize: number;
+  bulkIndexSize: number;
+  fullIndexName?: string;
+  knnVectorFieldConfig: {
+    mode: string;
+    dimension: number;
+    spaceType: string;
+    efConstruction: number;
+    efSearch: number;
+    m: number;
+    encoder?: {
+      name: string;
+      type: string;
+      clip: boolean;
+    };
+    compressionLevel?: string;
+  };
+}
+export interface MinioConfig {
+  endPoint: string;
+  port: number;
+  useSSL: boolean;
+  region: string;
+  defaultDatasetBucket: string;
+}
+export interface SemanticIndexerArguments extends MinionArguments {
+  opensearchApiURL: string;
+  embeddingApiURL: string;
+  semanticIndexerConfig: SemanticIndexerConfig;
+  minioConfig: MinioConfig;
+  minioAccessKey: string;
+  minioSecretKey: string;
+  registryReadonlyURL: string;
+}
+export declare function commonYargs(defaultPort: number, defaultInternalUrl: string): SemanticIndexerArguments;

package/dist/commonYargs.js

@@ -0,0 +1,61 @@
+import commonMinionYargs from "@magda/minion-framework/dist/commonYargs.js";
+import { require } from "@magda/esm-utils";
+export function commonYargs(defaultPort, defaultInternalUrl) {
+  let argv = commonMinionYargs(defaultPort, defaultInternalUrl, (y) => y.
+  option("semanticIndexerConfig", {
+    describe: "Semantic indexer configuration json",
+    type: "string",
+    coerce: (path) => {
+      if (!path) {
+        throw new Error("Semantic indexer configuration not found");
+      }
+      const config = path && require(path);
+      return config;
+    },
+    default: process.env.SEMANTIC_INDEXER_CONFIG_PATH || null
+  }).
+  option("minioConfig", {
+    describe: "Minio configuration json",
+    type: "string",
+    coerce: (path) => {
+      if (!path) {
+        throw new Error("Minio configuration not found");
+      }
+      const config = path && require(path);
+      return config;
+    },
+    default: process.env.MINIO_CONFIG_PATH || null
+  }).
+  option("minioAccessKey", {
+    describe: "The access key to your minio server.",
+    type: "string",
+    demand: true,
+    default: process.env.MINIO_ACCESS_KEY || ""
+  }).
+  option("minioSecretKey", {
+    describe: "The secret key to your minio server.",
+    type: "string",
+    demand: true,
+    default: process.env.MINIO_SECRET_KEY || ""
+  }).
+  option("opensearchApiURL", {
+    describe: "The URL of the OpenSearch API.",
+    type: "string",
+    default: process.env.OPENSEARCH_API_URL ||
+    "http://localhost:9200"
+  }).
+  option("embeddingApiURL", {
+    describe: "The URL of the embedding API.",
+    type: "string",
+    default: process.env.EMBEDDING_API_URL || "http://localhost:3000"
+  }).
+  option("registryReadonlyURL", {
+    describe: "The URL of the registry readonly API.",
+    type: "string",
+    default: process.env.REGISTRY_READ_ONLY_URL ||
+    "http://localhost:6101/v0"
+  }));
+  const { indexName, indexVersion } = argv.semanticIndexerConfig;
+  argv.semanticIndexerConfig.fullIndexName = `${indexName}-v${indexVersion}`;
+  return argv;
+}

package/dist/commonYargs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"commonYargs.js","sourceRoot":"","sources":["../src/commonYargs.ts"],"names":[],"mappings":"AACA,OAAO,iBAEN,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AA8C3C,MAAM,UAAU,WAAW,CACvB,WAAmB,EACnB,kBAA0B;IAE1B,IAAI,IAAI,GAAG,iBAAiB,CACxB,WAAW,EACX,kBAAkB,EAClB,CAAC,CAA8B,EAAE,EAAE,CAC/B,CAAC;SACI,MAAM,CAAC,uBAAuB,EAAE;QAC7B,QAAQ,EAAE,qCAAqC;QAC/C,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,CAAC,IAAY,EAAyB,EAAE;YAC5C,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,MAAM,IAAI,KAAK,CACX,0CAA0C,CAC7C,CAAC;YACN,CAAC;YACD,MAAM,MAAM,GACR,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;YAC1B,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,4BAA4B,IAAI,IAAI;KAC5D,CAAC;SACD,MAAM,CAAC,aAAa,EAAE;QACnB,QAAQ,EAAE,0BAA0B;QACpC,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,CAAC,IAAY,EAAe,EAAE;YAClC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACrD,CAAC;YACD,MAAM,MAAM,GAAgB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;YAClD,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI;KACjD,CAAC;SACD,MAAM,CAAC,gBAAgB,EAAE;QACtB,QAAQ,EAAE,sCAAsC;QAChD,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE;KAC9C,CAAC;SACD,MAAM,CAAC,gBAAgB,EAAE;QACtB,QAAQ,EAAE,sCAAsC;QAChD,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE;KAC9C,CAAC;SACD,MAAM,CAAC,kBAAkB,EAAE;QACxB,QAAQ,EAAE,gCAAgC;QAC1C,IAAI,EAAE,QAAQ;QACd,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,kBAAkB;YAC9B,uBAAuB;KAC9B,CAAC;SACD,MAAM,CAAC,iBAAiB,EAAE;QACvB,QAAQ,EAAE,+BAA+B;QACzC,IAAI,EAAE,QAAQ;QACd,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,uBAAuB;KAC/D,CAAC;SACD,MAAM,CAAC,qBAAqB,EAAE;QAC3B,QAAQ,EAAE,uCAAuC;QACjD,IAAI,EAAE,QAAQ;QACd,OAAO,EACH,OAAO,CAAC,GAAG,CAAC,sBAAsB;YAClC,0BAA0B;KACjC,CAAC,CACb,CAAC;IAEF,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,qBAAqB,CAAC;IAC/D,IAAI,CAAC,qBAAqB,CAAC,aAAa,GAAG,GAAG,SAAS,KAAK,YAAY,EAAE,CAAC;IAC3E,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/createEmbeddingText.d.ts

@@ -0,0 +1,18 @@
+import { Record } from "@magda/typescript-common/dist/generated/registry/api.js";
+import Registry from "@magda/typescript-common/dist/registry/AuthorizedRegistryClient.js";
+export type EmbeddingText = {
+  text: string;
+  subObjects?: Array<{
+    subObjectId?: string;
+    subObjectType?: string;
+    text: string;
+  }>;
+};
+export type CreateEmbeddingTextParams = {
+  record: Record;
+  format: string | null;
+  filePath: string | null;
+  url: string | null;
+  readonlyRegistry: Registry;
+};
+export type CreateEmbeddingText = (params: CreateEmbeddingTextParams) => Promise<EmbeddingText>;

package/dist/createEmbeddingText.js

@@ -0,0 +1 @@
+export {};

package/dist/createEmbeddingText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createEmbeddingText.js","sourceRoot":"","sources":["../src/createEmbeddingText.ts"],"names":[],"mappings":""}

package/dist/helpers.d.ts

@@ -0,0 +1 @@
+export declare function deleteTempFile(filePath: string): Promise<void>;

package/dist/helpers.js

@@ -0,0 +1,10 @@
+import fs from "fs";
+export async function deleteTempFile(filePath) {
+  if (fs.existsSync(filePath)) {
+    fs.unlink(filePath, (err) => {
+      if (err) {
+        console.error("Error deleting file");
+      }
+    });
+  }
+}

package/dist/helpers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.js","sourceRoot":"","sources":["../src/helpers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB;IACjD,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,IAAI,GAAG,EAAE,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;YACzC,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;AACL,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export { default as SemanticIndexerOptions } from "./semanticIndexerOptions.js";
+export { commonYargs } from "./commonYargs.js";
+export { ItemType } from "./indexSchema.js";
+export { CreateEmbeddingText, CreateEmbeddingTextParams, EmbeddingText } from "./createEmbeddingText.js";
+export { ChunkStrategyType, ChunkResult } from "./chunker.js";
+export { indexEmbeddingText } from "./indexEmbeddingText.js";
+import semanticIndexer from "./semanticIndexer.js";
+export default semanticIndexer;

package/dist/index.js

@@ -0,0 +1,4 @@
+export { commonYargs } from "./commonYargs.js";
+export { indexEmbeddingText } from "./indexEmbeddingText.js";
+import semanticIndexer from "./semanticIndexer.js";
+export default semanticIndexer;

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAW/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,eAAe,MAAM,sBAAsB,CAAC;AACnD,eAAe,eAAe,CAAC"}

package/dist/indexEmbeddingText.d.ts

@@ -0,0 +1,22 @@
+import EmbeddingApiClient from "@magda/typescript-common/dist/EmbeddingApiClient.js";
+import OpensearchApiClient from "@magda/typescript-common/dist/OpensearchApiClient.js";
+import { Chunker } from "./chunker.js";
+import { EmbeddingText } from "./createEmbeddingText.js";
+import SemanticIndexerOptions from "./semanticIndexerOptions.js";
+interface Metadata {
+  recordId: string;
+  parentRecordId?: string;
+  aspectId?: string;
+  fileFormat?: string;
+  subObjectId?: string;
+  subObjectType?: string;
+}
+export declare function indexEmbeddingText({ options, chunker, embeddingApiClient, opensearchApiClient, embeddingText, metadata
+
+
+
+
+
+
+}: {options: SemanticIndexerOptions;chunker: Chunker;embeddingApiClient: EmbeddingApiClient;opensearchApiClient: OpensearchApiClient;embeddingText: EmbeddingText;metadata: Metadata;}): Promise<void>;
+export {};

package/dist/indexEmbeddingText.js

@@ -0,0 +1,216 @@
+import { SkipError } from "./SkipError.js";
+import { buildSemanticIndexDocument } from "./indexSchema.js";
+import { v4 as uuidv4 } from "uuid";
+import { Readable, Transform, Writable } from "stream";
+import { pipeline } from "stream/promises";
+import retry from "@magda/typescript-common/dist/retry.js";
+class BuildDocumentTransform extends Transform {
+  params;
+  buffer = [];
+  constructor(params) {
+    super({ objectMode: true });
+    this.params = params;
+  }
+  async processBuffer() {
+    let embeddings;
+    try {
+      embeddings = await this.params.embeddingApiClient.get(this.buffer.map((c) => c.text));
+    }
+    catch (error) {
+      throw new SkipError(`Failed to get embeddings: ${error.message}`);
+    }
+    const documents = this.buffer.map((chunk, i) => buildSemanticIndexDocument({
+      recordId: this.params.metadata.recordId,
+      parentRecordId: this.params.metadata.parentRecordId,
+      aspectId: this.params.metadata.aspectId,
+      fileFormat: this.params.metadata.fileFormat,
+      subObjectId: this.params.metadata.subObjectId,
+      subObjectType: this.params.metadata.subObjectType,
+      itemType: this.params.options.itemType,
+      index_text_chunk: chunk.text,
+      embedding: embeddings[i],
+      only_one_index_text_chunk: this.params.totalChunks === 1,
+      index_text_chunk_length: chunk.length,
+      index_text_chunk_position: chunk.position,
+      index_text_chunk_overlap: chunk.overlap,
+      indexerId: this.params.options.id,
+      createTime: this.params.indexingStartTime,
+      updateTime: this.params.indexingStartTime
+    }));
+    for (let i = 0; i < documents.length; i++) {
+      this.push(documents[i]);
+    }
+    this.buffer = [];
+  }
+  async _transform(chunk, encoding, callback) {
+    this.buffer.push(chunk);
+    if (this.buffer.length >= this.params.bulkEmbeddingsSize) {
+      try {
+        await this.processBuffer();
+        callback();
+      }
+      catch (error) {
+        callback(error);
+      }
+    } else
+    {
+      callback();
+    }
+  }
+  async _flush(callback) {
+    if (this.buffer.length > 0) {
+      try {
+        await this.processBuffer();
+        callback();
+      }
+      catch (error) {
+        callback(error);
+      }
+    } else
+    {
+      callback();
+    }
+  }
+}
+class OpenSearchIndexStream extends Writable {
+  opensearchApiClient;
+  indexName;
+  bulkSize;
+  buffer;
+  constructor(opensearchApiClient, indexName, bulkSize) {
+    super({ objectMode: true });
+    this.opensearchApiClient = opensearchApiClient;
+    this.indexName = indexName;
+    this.bulkSize = bulkSize;
+    this.buffer = [];
+  }
+  async processBuffer() {
+    try {
+      await this.opensearchApiClient.bulkIndexDocument(this.indexName, this.buffer);
+      this.buffer = [];
+    }
+    catch (error) {
+      throw new SkipError(`Failed to index documents: ${error.message}`);
+    }
+  }
+  async _write(chunk, encoding, callback) {
+    this.buffer.push(chunk);
+    if (this.buffer.length >= this.bulkSize) {
+      try {
+        await this.processBuffer();
+        callback();
+      }
+      catch (error) {
+        callback(error);
+      }
+    } else
+    {
+      callback();
+    }
+  }
+  async _final(callback) {
+    if (this.buffer.length > 0) {
+      try {
+        await this.processBuffer();
+        callback();
+      }
+      catch (error) {
+        callback(error);
+      }
+    } else
+    {
+      callback();
+    }
+  }
+}
+export async function indexEmbeddingText({ options, chunker, embeddingApiClient, opensearchApiClient, embeddingText, metadata }) {
+  if (!embeddingText.text && !embeddingText.subObjects) {
+    throw new SkipError("No text or subObjects found to index.");
+  }
+  const textsToProcess = [];
+  if (embeddingText.text) {
+    textsToProcess.push({
+      text: embeddingText.text,
+      metadata: { ...metadata }
+    });
+  }
+  if (embeddingText.subObjects) {
+    for (const sub of embeddingText.subObjects) {
+      textsToProcess.push({
+        text: sub.text,
+        metadata: {
+          ...metadata,
+          subObjectId: sub.subObjectId || uuidv4(),
+          subObjectType: sub.subObjectType
+        }
+      });
+    }
+  }
+  await Promise.all(textsToProcess.map((item) => processSingleText({
+    options,
+    chunker,
+    embeddingApiClient,
+    opensearchApiClient,
+    metadata: item.metadata,
+    text: item.text
+  })));
+}
+async function processSingleText({ options, chunker, embeddingApiClient, opensearchApiClient, metadata, text }) {
+  const semanticIndexerConfig = options.argv.semanticIndexerConfig;
+  const bulkEmbeddingsSize = semanticIndexerConfig.bulkEmbeddingsSize || 50;
+  const bulkIndexSize = semanticIndexerConfig.bulkIndexSize || 1;
+  const indexName = semanticIndexerConfig.fullIndexName;
+  const chunks = await chunker.chunk(text);
+  if (!chunks || chunks.length === 0) {
+    throw new SkipError("No chunks generated from text.");
+  }
+  const indexingStartTime = new Date().toISOString();
+  const textChunkStream = Readable.from(chunks, { objectMode: true });
+  const buildDocumentTransform = new BuildDocumentTransform({
+    options,
+    embeddingApiClient,
+    bulkEmbeddingsSize,
+    totalChunks: chunks.length,
+    text,
+    metadata,
+    indexingStartTime
+  });
+  const openSearchStream = new OpenSearchIndexStream(opensearchApiClient, indexName, bulkIndexSize);
+  try {
+    await pipeline(textChunkStream, buildDocumentTransform, openSearchStream);
+    await deleteOldDocuments(opensearchApiClient, indexName, options.id, metadata.recordId, indexingStartTime, options.timeout || "1m", 3);
+  }
+  catch (error) {
+    throw new SkipError(`Failed to index documents: ${error.message}`);
+  }
+}
+async function deleteOldDocuments(opensearchApiClient, indexName, indexerId, recordId, beforeTimestamp, timeout, retries) {
+  const deleteQuery = {
+    bool: {
+      filter: [
+      { term: { indexerId: indexerId } },
+      { term: { recordId: recordId } },
+      { range: { createTime: { lt: beforeTimestamp } } }]
+
+    }
+  };
+  await retry(async () => {
+    const result = await opensearchApiClient.deleteByQuery({
+      index: indexName,
+      body: { query: deleteQuery },
+      wait_for_completion: true,
+      conflicts: "proceed",
+      timeout: timeout
+    });
+    const responseBody = result.body;
+    if (responseBody.version_conflicts !== 0) {
+      throw new Error(`${responseBody.version_conflicts} version conflicts`);
+    }
+    if (responseBody.timed_out) {
+      throw new Error(`timed out`);
+    }
+    return;
+  }, 1, retries, (error, left) => {
+    console.warn(`Failed to delete old documents: ${error.message}. remaining retries: ${left}`);
+  });
+}

package/dist/indexEmbeddingText.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexEmbeddingText.js","sourceRoot":"","sources":["../src/indexEmbeddingText.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAE3C,OAAO,KAAK,MAAM,sCAAsC,CAAC;AAWzD,MAAM,sBAAuB,SAAQ,SAAS;IAI9B;IAHJ,MAAM,GAAkB,EAAE,CAAC;IAEnC,YACY,MAQP;QAED,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAVpB,WAAM,GAAN,MAAM,CAQb;IAGL,CAAC;IAEO,KAAK,CAAC,aAAa;QACvB,IAAI,UAAsB,CAAC;QAC3B,IAAI,CAAC;YACD,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,GAAG,CACjD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CACjC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,SAAS,CACf,6BAA8B,KAAe,CAAC,OAAO,EAAE,CAC1D,CAAC;QACN,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAkB,EAAE,CAAS,EAAE,EAAE,CAChE,0BAA0B,CAAC;YACvB,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ;YACvC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc;YACnD,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ;YACvC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU;YAC3C,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW;YAC7C,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa;YACjD,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ;YACtC,gBAAgB,EAAE,KAAK,CAAC,IAAI;YAC5B,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;YACxB,yBAAyB,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,CAAC;YACxD,uBAAuB,EAAE,KAAK,CAAC,MAAM;YACrC,yBAAyB,EAAE,KAAK,CAAC,QAAQ;YACzC,wBAAwB,EAAE,KAAK,CAAC,OAAO;YACvC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YACjC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;YACzC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,iBAAiB;SAC5C,CAAC,CACL,CAAC;QAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAkB,EAAE,QAAgB,EAAE,QAAkB;QACrE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;YACvD,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;gBAC3B,QAAQ,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,QAAQ,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAkB;QAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;gBAC3B,QAAQ,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,QAAQ,EAAE,CAAC;QACf,CAAC;IACL,CAAC;CACJ;AAED,MAAM,qBAAsB,SAAQ,QAAQ;IAI5B;IACA;IACA;IALJ,MAAM,CAAQ;IAEtB,YACY,mBAAwC,EACxC,SAAiB,EACjB,QAAgB;QAExB,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAJpB,wBAAmB,GAAnB,mBAAmB,CAAqB;QACxC,cAAS,GAAT,SAAS,CAAQ;QACjB,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,aAAa;QACvB,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAC5C,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,MAAM,CACd,CAAC;YACF,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,SAAS,CACf,8BAA+B,KAAe,CAAC,OAAO,EAAE,CAC3D,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAU,EAAE,QAAgB,EAAE,QAAkB;QACzD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAExB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtC,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;gBAC3B,QAAQ,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,QAAQ,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAkB;QAC3B,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;gBAC3B,QAAQ,EAAE,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,QAAQ,EAAE,CAAC;QACf,CAAC;IACL,CAAC;CACJ;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACrC,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,mBAAmB,EACnB,aAAa,EACb,QAAQ,EAQX;IACG,IAAI,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;QACnD,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,cAAc,GAAgD,EAAE,CAAC;IAEvE,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC;QACrB,cAAc,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE;SAC5B,CAAC,CAAC;IACP,CAAC;IAED,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,UAAU,EAAE,CAAC;YACzC,cAAc,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,QAAQ,EAAE;oBACN,GAAG,QAAQ;oBACX,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,MAAM,EAAE;oBACxC,aAAa,EAAE,GAAG,CAAC,aAAa;iBACnC;aACJ,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CACb,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACxB,iBAAiB,CAAC;QACd,OAAO;QACP,OAAO;QACP,kBAAkB;QAClB,mBAAmB;QACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,IAAI,EAAE,IAAI,CAAC,IAAI;KAClB,CAAC,CACL,CACJ,CAAC;AACN,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,EAC7B,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,mBAAmB,EACnB,QAAQ,EACR,IAAI,EAQP;IACG,MAAM,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC,qBAAqB,CAAC;IACjE,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,kBAAkB,IAAI,EAAE,CAAC;IAC1E,MAAM,aAAa,GAAG,qBAAqB,CAAC,aAAa,IAAI,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,qBAAqB,CAAC,aAAa,CAAC;IAEtD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,SAAS,CAAC,gCAAgC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACnD,MAAM,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IACpE,MAAM,sBAAsB,GAAG,IAAI,sBAAsB,CAAC;QACtD,OAAO;QACP,kBAAkB;QAClB,kBAAkB;QAClB,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,IAAI;QACJ,QAAQ;QACR,iBAAiB;KACpB,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG,IAAI,qBAAqB,CAC9C,mBAAmB,EACnB,SAAS,EACT,aAAa,CAChB,CAAC;IAEF,IAAI,CAAC;QACD,MAAM,QAAQ,CACV,eAAe,EACf,sBAAsB,EACtB,gBAAgB,CACnB,CAAC;QAEF,MAAM,kBAAkB,CACpB,mBAAmB,EACnB,SAAS,EACT,OAAO,CAAC,EAAE,EACV,QAAQ,CAAC,QAAQ,EACjB,iBAAiB,EACjB,OAAO,CAAC,OAAO,IAAI,IAAI,EACvB,CAAC,CACJ,CAAC;IACN,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,IAAI,SAAS,CACf,8BAA+B,KAAe,CAAC,OAAO,EAAE,CAC3D,CAAC;IACN,CAAC;AACL,CAAC;AAED,KAAK,UAAU,kBAAkB,CAC7B,mBAAwC,EACxC,SAAiB,EACjB,SAAiB,EACjB,QAAgB,EAChB,eAAuB,EACvB,OAAe,EACf,OAAe;IAEf,MAAM,WAAW,GAAG;QAChB,IAAI,EAAE;YACF,MAAM,EAAE;gBACJ,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE;gBAClC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE;gBAChC,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE;aACrD;SACJ;KACJ,CAAC;IAEF,MAAM,KAAK,CACP,KAAK,IAAI,EAAE;QACP,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,aAAa,CAAC;YACnD,KAAK,EAAE,SAAS;YAChB,IAAI,EAAE,EAAE,KAAK,EAAE,WAAkB,EAAE;YACnC,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,OAAO;SACnB,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,CAAC,IAAgC,CAAC;QAC7D,IAAI,YAAY,CAAC,iBAAiB,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACX,GAAG,YAAY,CAAC,iBAAiB,oBAAoB,CACxD,CAAC;QACN,CAAC;QACD,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC;QACjC,CAAC;QACD,OAAO;IACX,CAAC,EACD,CAAC,EACD,OAAO,EACP,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACZ,OAAO,CAAC,IAAI,CACR,mCACK,KAAe,CAAC,OACrB,wBAAwB,IAAI,EAAE,CACjC,CAAC;IACN,CAAC,CACJ,CAAC;AACN,CAAC"}