npm - node-es-transformer - Versions diffs - 1.0.2 → 1.1.0 - Mend

node-es-transformer 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

package/README.md +58 -4
package/dist/node-es-transformer.cjs.js +279 -81
package/dist/node-es-transformer.cjs.js.map +1 -1
package/dist/node-es-transformer.esm.js +280 -82
package/dist/node-es-transformer.esm.js.map +1 -1
package/index.d.ts +58 -1
package/package.json +2 -1

package/dist/node-es-transformer.esm.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"node-es-transformer.esm.js","sources":["../src/_constants.js","../src/_create-mapping.js","../src/_file-reader.js","../src/_index-queue.js","../src/_index-reader.js","../src/_stream-reader.js","../src/main.js"],"sourcesContent":["// In earlier versions this was used to set the number of docs to index in a\n// single bulk request. Since we switched to use the helpers.bulk() method from\n// the ES client, this now translates to the `flushBytes` option of the helper.\n// However, for kind of a backwards compability with the old values, this uses\n// KBytes instead of Bytes. It will be multiplied by 1024 in the index queue.\nexport const DEFAULT_BUFFER_SIZE = 5120;\n\n// The default number of docs to fetch in a single search request when reindexing.\nexport const DEFAULT_SEARCH_SIZE = 1000;\n","export default function createMappingFactory({\n sourceClient,\n sourceIndexName,\n targetClient,\n targetIndexName,\n mappings,\n mappingsOverride,\n indexMappingTotalFieldsLimit,\n verbose,\n deleteIndex,\n pipeline,\n}) {\n return async () => {\n let targetMappings = mappingsOverride ? undefined : mappings;\n\n if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {\n try {\n const mapping = await sourceClient.indices.getMapping({\n index: sourceIndexName,\n });\n if (mapping[sourceIndexName]) {\n targetMappings = mapping[sourceIndexName].mappings;\n } else {\n const allMappings = Object.values(mapping);\n if (allMappings.length > 0) {\n targetMappings = Object.values(mapping)[0].mappings;\n }\n }\n } catch (err) {\n console.log('Error reading source mapping', err);\n return;\n }\n }\n\n if (typeof targetMappings === 'object' && targetMappings !== null) {\n if (mappingsOverride) {\n targetMappings = {\n ...targetMappings,\n properties: {\n ...targetMappings.properties,\n ...mappings,\n },\n };\n }\n\n try {\n const indexExists = await targetClient.indices.exists({ index: targetIndexName });\n\n if (indexExists === true && deleteIndex === true) {\n await targetClient.indices.delete({ index: targetIndexName });\n }\n\n if (indexExists === false \|\| deleteIndex === true) {\n const resp = await targetClient.indices.create({\n index: targetIndexName,\n mappings: targetMappings,\n ...(pipeline !== undefined\n ? {\n settings: {\n index: {\n default_pipeline: pipeline,\n },\n },\n }\n : {}),\n ...(indexMappingTotalFieldsLimit !== undefined\n ? {\n settings: {\n 'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,\n 'index.number_of_shards': 1,\n 'index.number_of_replicas': 0,\n },\n }\n : {}),\n });\n if (verbose) console.log('Created target mapping', resp);\n }\n } catch (err) {\n console.log('Error creating target mapping', err);\n }\n }\n };\n}\n","import fs from 'fs';\nimport es from 'event-stream';\nimport { globSync } from 'glob';\nimport split from 'split2';\n\nexport default function fileReaderFactory(indexer, fileName, transform, splitRegex, verbose) {\n function startIndex(files) {\n let finished = false;\n\n const file = files.shift();\n const s = fs\n .createReadStream(file)\n .pipe(split(splitRegex))\n .pipe(\n es\n .mapSync(line => {\n try {\n // skip empty lines\n if (line === '') {\n return;\n }\n\n const doc =\n typeof transform === 'function'\n ? JSON.stringify(transform(JSON.parse(line)))\n : line;\n\n // if doc is undefined we'll skip indexing it\n if (typeof doc === 'undefined') {\n s.resume();\n return;\n }\n\n // the transform callback may return an array of docs so we can emit\n // multiple docs from a single line\n if (Array.isArray(doc)) {\n doc.forEach(d => indexer.add(d));\n return;\n }\n\n indexer.add(doc);\n } catch (e) {\n console.log('error', e);\n }\n })\n .on('error', err => {\n console.log('Error while reading file.', err);\n })\n .on('end', () => {\n if (verbose) console.log('Read entire file: ', file);\n if (files.length > 0) {\n startIndex(files);\n return;\n }\n\n indexer.finish();\n finished = true;\n }),\n );\n\n indexer.queueEmitter.on('pause', () => {\n if (finished) return;\n s.pause();\n });\n\n indexer.queueEmitter.on('resume', () => {\n if (finished) return;\n s.resume();\n });\n }\n\n return () => {\n try {\n const files = globSync(fileName);\n startIndex(files);\n } catch (error) {\n console.log('Error matching files:', error);\n }\n };\n}\n","import { Readable } from 'stream';\n\nimport { DEFAULT_BUFFER_SIZE } from './_constants';\n\nconst EventEmitter = require('events');\n\nconst queueEmitter = new EventEmitter();\n\nconst parallelCalls = 5;\n\n// a simple helper queue to bulk index documents\nexport default function indexQueueFactory({\n targetClient: client,\n targetIndexName,\n bufferSize = DEFAULT_BUFFER_SIZE,\n skipHeader = false,\n}) {\n let docsPerSecond = 0;\n\n const flushBytes = bufferSize * 1024; // Convert KB to Bytes\n const highWaterMark = flushBytes * parallelCalls;\n\n // Create a Readable stream\n const stream = new Readable({\n read() {}, // Implement read but we manage pushing manually\n highWaterMark, // Buffer size for backpressure management\n });\n\n async function* ndjsonStreamIterator(readableStream) {\n let buffer = ''; // To hold the incomplete data\n let skippedHeader = false;\n\n try {\n // Iterate over the stream using async iteration\n for await (const chunk of readableStream) {\n buffer += chunk.toString(); // Accumulate the chunk data in the buffer\n\n // Split the buffer into lines (NDJSON items)\n const lines = buffer.split('\\n');\n\n // The last line might be incomplete, so hold it back in the buffer\n buffer = lines.pop();\n\n // Yield each complete JSON object\n for (const line of lines) {\n if (line.trim()) {\n try {\n if (!skipHeader \|\| (skipHeader && !skippedHeader)) {\n yield JSON.parse(line); // Parse and yield the JSON object\n skippedHeader = true;\n }\n } catch (err) {\n // Handle JSON parse errors if necessary\n console.error('Failed to parse JSON:', err);\n }\n }\n }\n }\n\n // Handle any remaining data in the buffer after the stream ends\n if (buffer.trim()) {\n try {\n yield JSON.parse(buffer);\n } catch (err) {\n console.error('Failed to parse final JSON:', err);\n }\n }\n } finally {\n // Ensure the stream is properly cleaned up if the iterator is terminated early\n if (!readableStream.destroyed) {\n readableStream.destroy();\n }\n }\n }\n\n let finished = false;\n let drainListener = null;\n\n // Async IIFE to start bulk indexing\n (async () => {\n const interval = setInterval(() => {\n queueEmitter.emit('docsPerSecond', docsPerSecond);\n docsPerSecond = 0;\n }, 1000);\n\n try {\n await client.helpers.bulk({\n concurrency: parallelCalls,\n flushBytes,\n flushInterval: 1000,\n refreshOnCompletion: true,\n datasource: ndjsonStreamIterator(stream),\n onDocument(doc) {\n docsPerSecond++;\n return {\n index: { _index: targetIndexName },\n };\n },\n });\n } catch (error) {\n console.error('Error during bulk indexing:', error);\n throw error;\n } finally {\n // Clean up interval\n clearInterval(interval);\n\n // Remove drain listener if it exists\n if (drainListener) {\n stream.removeListener('drain', drainListener);\n drainListener = null;\n }\n\n // Remove all listeners from stream\n stream.removeAllListeners();\n\n // Properly destroy the stream to prevent open handles\n if (!stream.destroyed) {\n stream.destroy();\n }\n\n // Emit finish and clean up queue emitter listeners\n queueEmitter.emit('finish');\n queueEmitter.removeAllListeners();\n }\n })();\n\n return {\n add: doc => {\n if (finished) {\n throw new Error('Unexpected doc added after indexer should finish.');\n }\n\n const canContinue = stream.push(`${JSON.stringify(doc)}\\n`);\n if (!canContinue) {\n queueEmitter.emit('pause');\n\n // Store the listener so we can clean it up later\n drainListener = () => {\n queueEmitter.emit('resume');\n };\n stream.once('drain', drainListener);\n }\n },\n finish: () => {\n finished = true;\n stream.push(null);\n },\n queueEmitter,\n };\n}\n","import cliProgress from 'cli-progress';\n\nimport { DEFAULT_SEARCH_SIZE } from './_constants';\n\n// create a new progress bar instance and use shades_classic theme\nconst progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\nexport default function indexReaderFactory(\n indexer,\n sourceIndexName,\n transform,\n client,\n query,\n searchSize = DEFAULT_SEARCH_SIZE,\n populatedFields = false,\n) {\n return async function indexReader() {\n let docsNum = 0;\n let scrollId;\n let finished = false;\n let readActive = false;\n let backPressurePause = false;\n\n async function fetchPopulatedFields() {\n try {\n // Get all populated fields from the index\n const response = await client.fieldCaps(\n {\n index: sourceIndexName,\n fields: '',\n include_empty_fields: false,\n filters: '-metadata',\n },\n { maxRetries: 0 },\n );\n\n return Object.keys(response.fields);\n } catch (e) {\n console.log('error', e);\n }\n }\n\n function search(fields) {\n return client.search({\n index: sourceIndexName,\n scroll: '600s',\n size: searchSize,\n query,\n ...(fields ? { _source: fields } : {}),\n });\n }\n\n function scroll(id) {\n return client.scroll({\n scroll_id: id,\n scroll: '600s',\n });\n }\n\n let fieldsWithData;\n\n // identify populated fields\n if (populatedFields) {\n fieldsWithData = await fetchPopulatedFields();\n }\n\n await fetchNextResponse();\n\n function processHit(hit) {\n docsNum += 1;\n try {\n const doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle\n\n // if doc is undefined we'll skip indexing it\n if (typeof doc === 'undefined') {\n return;\n }\n\n // the transform callback may return an array of docs so we can emit\n // multiple docs from a single line\n if (Array.isArray(doc)) {\n doc.forEach(d => indexer.add(d));\n return;\n }\n\n indexer.add(doc);\n } catch (e) {\n console.log('error', e);\n }\n }\n\n async function fetchNextResponse() {\n readActive = true;\n\n const sc = scrollId ? await scroll(scrollId) : await search(fieldsWithData);\n\n if (!scrollId) {\n progressBar.start(sc.hits.total.value, 0);\n }\n\n scrollId = sc._scroll_id;\n readActive = false;\n\n processResponse(sc);\n }\n\n async function processResponse(response) {\n // collect the docs from this response\n response.hits.hits.forEach(processHit);\n\n progressBar.update(docsNum);\n\n // check to see if we have collected all of the docs\n if (response.hits.total.value === docsNum) {\n indexer.finish();\n return;\n }\n\n if (!backPressurePause) {\n await fetchNextResponse();\n }\n }\n\n indexer.queueEmitter.on('pause', async () => {\n backPressurePause = true;\n });\n\n indexer.queueEmitter.on('resume', async () => {\n backPressurePause = false;\n\n if (readActive \|\| finished) {\n return;\n }\n\n await fetchNextResponse();\n });\n\n indexer.queueEmitter.on('finish', () => {\n finished = true;\n progressBar.stop();\n });\n };\n}\n","import es from 'event-stream';\nimport split from 'split2';\n\nexport default function streamReaderFactory(indexer, stream, transform, splitRegex, verbose) {\n function startIndex() {\n let finished = false;\n\n const s = stream.pipe(split(splitRegex)).pipe(\n es\n .mapSync(line => {\n try {\n // skip empty lines\n if (line === '') {\n return;\n }\n\n const doc =\n typeof transform === 'function' ? JSON.stringify(transform(JSON.parse(line))) : line;\n\n // if doc is undefined we'll skip indexing it\n if (typeof doc === 'undefined') {\n s.resume();\n return;\n }\n\n // the transform callback may return an array of docs so we can emit\n // multiple docs from a single line\n if (Array.isArray(doc)) {\n doc.forEach(d => indexer.add(d));\n return;\n }\n\n indexer.add(doc);\n } catch (e) {\n console.log('error', e);\n }\n })\n .on('error', err => {\n console.log('Error while reading stream.', err);\n })\n .on('end', () => {\n if (verbose) console.log('Read entire stream.');\n indexer.finish();\n finished = true;\n }),\n );\n\n indexer.queueEmitter.on('pause', () => {\n if (finished) return;\n s.pause();\n });\n\n indexer.queueEmitter.on('resume', () => {\n if (finished) return;\n s.resume();\n });\n }\n\n return () => {\n startIndex();\n };\n}\n","import elasticsearch9 from 'es9';\nimport elasticsearch8 from 'es8';\n\nimport { DEFAULT_BUFFER_SIZE, DEFAULT_SEARCH_SIZE } from './_constants';\nimport createMappingFactory from './_create-mapping';\nimport fileReaderFactory from './_file-reader';\nimport indexQueueFactory from './_index-queue';\nimport indexReaderFactory from './_index-reader';\nimport streamReaderFactory from './_stream-reader';\n\n/\n Detect Elasticsearch version by querying the cluster\n /\nasync function detectElasticsearchVersion(config) {\n try {\n // Try with v9 client first (most common for new setups)\n const testClient = new elasticsearch9.Client(config);\n const info = await testClient.info();\n const version = info.version?.number;\n await testClient.close();\n\n if (version) {\n const majorVersion = parseInt(version.split('.')[0], 10);\n return majorVersion;\n }\n } catch (e) {\n // If v9 client fails, try v8 client\n try {\n const testClient = new elasticsearch8.Client(config);\n const info = await testClient.info();\n const version = info.version?.number;\n await testClient.close();\n\n if (version) {\n const majorVersion = parseInt(version.split('.')[0], 10);\n return majorVersion;\n }\n } catch (e2) {\n // Could not detect version\n }\n }\n\n // Default to v9 if detection fails\n return 9;\n}\n\n/\n Create or validate an Elasticsearch client\n * @param {Object\|Client} clientOrConfig - Either a client instance or config object\n * @param {Object} defaultConfig - Default configuration to use if creating a new client\n * @param {number} [forceVersion] - Force a specific ES client version (8 or 9)\n */\nasync function getOrCreateClient(clientOrConfig, defaultConfig, forceVersion) {\n // If already a client instance, return it\n if (clientOrConfig && typeof clientOrConfig.info === 'function') {\n return clientOrConfig;\n }\n\n const config = clientOrConfig \|\| defaultConfig;\n\n // If version is forced, use the specified client\n if (forceVersion === 8) {\n return new elasticsearch8.Client(config);\n } else if (forceVersion === 9) {\n return new elasticsearch9.Client(config);\n }\n\n // Auto-detect version\n const majorVersion = await detectElasticsearchVersion(config);\n\n if (majorVersion >= 9) {\n return new elasticsearch9.Client(config);\n } else {\n return new elasticsearch8.Client(config);\n }\n}\n\nexport default async function transformer({\n deleteIndex = false,\n sourceClient: sourceClientInput,\n targetClient: targetClientInput,\n sourceClientConfig,\n targetClientConfig,\n sourceClientVersion,\n targetClientVersion,\n bufferSize = DEFAULT_BUFFER_SIZE,\n searchSize = DEFAULT_SEARCH_SIZE,\n stream,\n fileName,\n splitRegex = /\\n/,\n sourceIndexName,\n targetIndexName,\n mappings,\n mappingsOverride = false,\n indexMappingTotalFieldsLimit,\n pipeline,\n populatedFields = false,\n query,\n skipHeader = false,\n transform,\n verbose = true,\n}) {\n if (typeof targetIndexName === 'undefined') {\n throw Error('targetIndexName must be specified.');\n }\n\n const defaultClientConfig = {\n node: process.env.ELASTICSEARCH_URL \|\| 'http://localhost:9200',\n };\n\n // Support both old (config) and new (client instance) patterns\n const sourceClient = await getOrCreateClient(\n sourceClientInput \|\| sourceClientConfig,\n defaultClientConfig,\n sourceClientVersion,\n );\n\n const targetClient = await getOrCreateClient(\n targetClientInput \|\| targetClientConfig \|\| sourceClientInput \|\| sourceClientConfig,\n defaultClientConfig,\n targetClientVersion,\n );\n\n const createMapping = createMappingFactory({\n sourceClient,\n sourceIndexName,\n targetClient,\n targetIndexName,\n mappings,\n mappingsOverride,\n indexMappingTotalFieldsLimit,\n verbose,\n deleteIndex,\n pipeline,\n });\n const indexer = indexQueueFactory({\n targetClient,\n targetIndexName,\n bufferSize,\n skipHeader,\n verbose,\n });\n\n function getReader() {\n if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {\n throw Error('Only either one of fileName or sourceIndexName can be specified.');\n }\n\n if (\n (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') \|\|\n (typeof fileName !== 'undefined' && typeof stream !== 'undefined') \|\|\n (typeof sourceIndexName !== 'undefined' && typeof stream !== 'undefined')\n ) {\n throw Error('Only one of fileName, sourceIndexName, or stream can be specified.');\n }\n\n if (typeof fileName !== 'undefined') {\n return fileReaderFactory(indexer, fileName, transform, splitRegex, verbose);\n }\n\n if (typeof sourceIndexName !== 'undefined') {\n return indexReaderFactory(\n indexer,\n sourceIndexName,\n transform,\n sourceClient,\n query,\n searchSize,\n populatedFields,\n );\n }\n\n if (typeof stream !== 'undefined') {\n return streamReaderFactory(indexer, stream, transform, splitRegex, verbose);\n }\n\n return null;\n }\n\n const reader = getReader();\n\n try {\n const indexExists = await targetClient.indices.exists({ index: targetIndexName });\n\n if (indexExists === false) {\n await createMapping();\n reader();\n } else if (deleteIndex === true) {\n await targetClient.indices.delete({ index: targetIndexName });\n await createMapping();\n reader();\n } else {\n reader();\n }\n } catch (error) {\n console.error('Error checking index existence:', error);\n } finally {\n // targetClient.close();\n }\n\n return { events: indexer.queueEmitter };\n}\n"],"names":["DEFAULT_BUFFER_SIZE","DEFAULT_SEARCH_SIZE","createMappingFactory","sourceClient","sourceIndexName","targetClient","targetIndexName","mappings","mappingsOverride","indexMappingTotalFieldsLimit","verbose","deleteIndex","pipeline","targetMappings","undefined","mapping","indices","getMapping","index","allMappings","Object","values","length","err","console","log","properties","indexExists","exists","delete","resp","create","settings","default_pipeline","fileReaderFactory","indexer","fileName","transform","splitRegex","startIndex","files","finished","file","shift","s","fs","createReadStream","pipe","split","es","mapSync","line","doc","JSON","stringify","parse","resume","Array","isArray","forEach","d","add","e","on","finish","queueEmitter","pause","globSync","error","EventEmitter","require","parallelCalls","indexQueueFactory","client","bufferSize","skipHeader","docsPerSecond","flushBytes","highWaterMark","stream","Readable","read","ndjsonStreamIterator","readableStream","buffer","skippedHeader","chunk","toString","lines","pop","trim","destroyed","destroy","drainListener","interval","setInterval","emit","helpers","bulk","concurrency","flushInterval","refreshOnCompletion","datasource","onDocument","_index","clearInterval","removeListener","removeAllListeners","Error","canContinue","push","once","progressBar","cliProgress","SingleBar","Presets","shades_classic","indexReaderFactory","query","searchSize","populatedFields","indexReader","docsNum","scrollId","readActive","backPressurePause","fetchPopulatedFields","response","fieldCaps","fields","include_empty_fields","filters","maxRetries","keys","search","scroll","size","_source","id","scroll_id","fieldsWithData","fetchNextResponse","processHit","hit","sc","start","hits","total","value","_scroll_id","processResponse","update","stop","streamReaderFactory","detectElasticsearchVersion","config","testClient","elasticsearch9","Client","info","version","number","close","majorVersion","parseInt","elasticsearch8","e2","getOrCreateClient","clientOrConfig","defaultConfig","forceVersion","transformer","sourceClientInput","targetClientInput","sourceClientConfig","targetClientConfig","sourceClientVersion","targetClientVersion","defaultClientConfig","node","process","env","ELASTICSEARCH_URL","createMapping","getReader","reader","events"],"mappings":";;;;;;;;;AAAA;AACA;AACA;AACA;AACA;AACO,MAAMA,mBAAmB,GAAG,IAAI;;AAEvC;AACO,MAAMC,mBAAmB,GAAG,IAAI;;ACRxB,SAASC,oBAAoBA,CAAC;EAC3CC,YAAY;EACZC,eAAe;EACfC,YAAY;EACZC,eAAe;EACfC,QAAQ;EACRC,gBAAgB;EAChBC,4BAA4B;EAC5BC,OAAO;EACPC,WAAW;AACXC,EAAAA;AACF,CAAC,EAAE;AACD,EAAA,OAAO,YAAY;AACjB,IAAA,IAAIC,cAAc,GAAGL,gBAAgB,GAAGM,SAAS,GAAGP,QAAQ;IAE5D,IAAIJ,YAAY,IAAIC,eAAe,IAAI,OAAOS,cAAc,KAAK,WAAW,EAAE;MAC5E,IAAI;QACF,MAAME,OAAO,GAAG,MAAMZ,YAAY,CAACa,OAAO,CAACC,UAAU,CAAC;AACpDC,UAAAA,KAAK,EAAEd;AACT,SAAC,CAAC;AACF,QAAA,IAAIW,OAAO,CAACX,eAAe,CAAC,EAAE;AAC5BS,UAAAA,cAAc,GAAGE,OAAO,CAACX,eAAe,CAAC,CAACG,QAAQ;AACpD,QAAA,CAAC,MAAM;AACL,UAAA,MAAMY,WAAW,GAAGC,MAAM,CAACC,MAAM,CAACN,OAAO,CAAC;AAC1C,UAAA,IAAII,WAAW,CAACG,MAAM,GAAG,CAAC,EAAE;YAC1BT,cAAc,GAAGO,MAAM,CAACC,MAAM,CAACN,OAAO,CAAC,CAAC,CAAC,CAAC,CAACR,QAAQ;AACrD,UAAA;AACF,QAAA;MACF,CAAC,CAAC,OAAOgB,GAAG,EAAE;AACZC,QAAAA,OAAO,CAACC,GAAG,CAAC,8BAA8B,EAAEF,GAAG,CAAC;AAChD,QAAA;AACF,MAAA;AACF,IAAA;IAEA,IAAI,OAAOV,cAAc,KAAK,QAAQ,IAAIA,cAAc,KAAK,IAAI,EAAE;AACjE,MAAA,IAAIL,gBAAgB,EAAE;AACpBK,QAAAA,cAAc,GAAG;AACf,UAAA,GAAGA,cAAc;AACjBa,UAAAA,UAAU,EAAE;YACV,GAAGb,cAAc,CAACa,UAAU;YAC5B,GAAGnB;AACL;SACD;AACH,MAAA;MAEA,IAAI;QACF,MAAMoB,WAAW,GAAG,MAAMtB,YAAY,CAACW,OAAO,CAACY,MAAM,CAAC;AAAEV,UAAAA,KAAK,EAAEZ;AAAgB,SAAC,CAAC;AAEjF,QAAA,IAAIqB,WAAW,KAAK,IAAI,IAAIhB,WAAW,KAAK,IAAI,EAAE;AAChD,UAAA,MAAMN,YAAY,CAACW,OAAO,CAACa,MAAM,CAAC;AAAEX,YAAAA,KAAK,EAAEZ;AAAgB,WAAC,CAAC;AAC/D,QAAA;AAEA,QAAA,IAAIqB,WAAW,KAAK,KAAK,IAAIhB,WAAW,KAAK,IAAI,EAAE;UACjD,MAAMmB,IAAI,GAAG,MAAMzB,YAAY,CAACW,OAAO,CAACe,MAAM,CAAC;AAC7Cb,YAAAA,KAAK,EAAEZ,eAAe;AACtBC,YAAAA,QAAQ,EAAEM,cAAc;YACxB,IAAID,QAAQ,KAAKE,SAAS,GACtB;AACEkB,cAAAA,QAAQ,EAAE;AACRd,gBAAAA,KAAK,EAAE;AACLe,kBAAAA,gBAAgB,EAAErB;AACpB;AACF;aACD,GACD,EAAE,CAAC;YACP,IAAIH,4BAA4B,KAAKK,SAAS,GAC1C;AACEkB,cAAAA,QAAQ,EAAE;AACR,gBAAA,kCAAkC,EAAEvB,4BAA4B;AAChE,gBAAA,wBAAwB,EAAE,CAAC;AAC3B,gBAAA,0BAA0B,EAAE;AAC9B;aACD,GACD,EAAE;AACR,WAAC,CAAC;UACF,IAAIC,OAAO,EAAEc,OAAO,CAACC,GAAG,CAAC,wBAAwB,EAAEK,IAAI,CAAC;AAC1D,QAAA;MACF,CAAC,CAAC,OAAOP,GAAG,EAAE;AACZC,QAAAA,OAAO,CAACC,GAAG,CAAC,+BAA+B,EAAEF,GAAG,CAAC;AACnD,MAAA;AACF,IAAA;EACF,CAAC;AACH;;AC7Ee,SAASW,iBAAiBA,CAACC,OAAO,EAAEC,QAAQ,EAAEC,SAAS,EAAEC,UAAU,EAAE5B,OAAO,EAAE;EAC3F,SAAS6B,UAAUA,CAACC,KAAK,EAAE;IACzB,IAAIC,QAAQ,GAAG,KAAK;AAEpB,IAAA,MAAMC,IAAI,GAAGF,KAAK,CAACG,KAAK,EAAE;IAC1B,MAAMC,CAAC,GAAGC,EAAE,CACTC,gBAAgB,CAACJ,IAAI,CAAC,CACtBK,IAAI,CAACC,KAAK,CAACV,UAAU,CAAC,CAAC,CACvBS,IAAI,CACHE,EAAE,CACCC,OAAO,CAACC,IAAI,IAAI;MACf,IAAI;AACF;QACA,IAAIA,IAAI,KAAK,EAAE,EAAE;AACf,UAAA;AACF,QAAA;QAEA,MAAMC,GAAG,GACP,OAAOf,SAAS,KAAK,UAAU,GAC3BgB,IAAI,CAACC,SAAS,CAACjB,SAAS,CAACgB,IAAI,CAACE,KAAK,CAACJ,IAAI,CAAC,CAAC,CAAC,GAC3CA,IAAI;;AAEV;AACA,QAAA,IAAI,OAAOC,GAAG,KAAK,WAAW,EAAE;UAC9BR,CAAC,CAACY,MAAM,EAAE;AACV,UAAA;AACF,QAAA;;AAEA;AACA;AACA,QAAA,IAAIC,KAAK,CAACC,OAAO,CAACN,GAAG,CAAC,EAAE;UACtBA,GAAG,CAACO,OAAO,CAACC,CAAC,IAAIzB,OAAO,CAAC0B,GAAG,CAACD,CAAC,CAAC,CAAC;AAChC,UAAA;AACF,QAAA;AAEAzB,QAAAA,OAAO,CAAC0B,GAAG,CAACT,GAAG,CAAC;MAClB,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtC,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqC,CAAC,CAAC;AACzB,MAAA;AACF,IAAA,CAAC,CAAC,CACDC,EAAE,CAAC,OAAO,EAAExC,GAAG,IAAI;AAClBC,MAAAA,OAAO,CAACC,GAAG,CAAC,2BAA2B,EAAEF,GAAG,CAAC;AAC/C,IAAA,CAAC,CAAC,CACDwC,EAAE,CAAC,KAAK,EAAE,MAAM;MACf,IAAIrD,OAAO,EAAEc,OAAO,CAACC,GAAG,CAAC,oBAAoB,EAAEiB,IAAI,CAAC;AACpD,MAAA,IAAIF,KAAK,CAAClB,MAAM,GAAG,CAAC,EAAE;QACpBiB,UAAU,CAACC,KAAK,CAAC;AACjB,QAAA;AACF,MAAA;MAEAL,OAAO,CAAC6B,MAAM,EAAE;AAChBvB,MAAAA,QAAQ,GAAG,IAAI;AACjB,IAAA,CAAC,CACL,CAAC;AAEHN,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,OAAO,EAAE,MAAM;AACrC,MAAA,IAAItB,QAAQ,EAAE;MACdG,CAAC,CAACsB,KAAK,EAAE;AACX,IAAA,CAAC,CAAC;AAEF/B,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,QAAQ,EAAE,MAAM;AACtC,MAAA,IAAItB,QAAQ,EAAE;MACdG,CAAC,CAACY,MAAM,EAAE;AACZ,IAAA,CAAC,CAAC;AACJ,EAAA;AAEA,EAAA,OAAO,MAAM;IACX,IAAI;AACF,MAAA,MAAMhB,KAAK,GAAG2B,QAAQ,CAAC/B,QAAQ,CAAC;MAChCG,UAAU,CAACC,KAAK,CAAC;IACnB,CAAC,CAAC,OAAO4B,KAAK,EAAE;AACd5C,MAAAA,OAAO,CAACC,GAAG,CAAC,uBAAuB,EAAE2C,KAAK,CAAC;AAC7C,IAAA;EACF,CAAC;AACH;;AC3EA,MAAMC,YAAY,GAAGC,OAAO,CAAC,QAAQ,CAAC;AAEtC,MAAML,YAAY,GAAG,IAAII,YAAY,EAAE;AAEvC,MAAME,aAAa,GAAG,CAAC;;AAEvB;AACe,SAASC,iBAAiBA,CAAC;AACxCnE,EAAAA,YAAY,EAAEoE,MAAM;EACpBnE,eAAe;AACfoE,EAAAA,UAAU,GAAG1E,mBAAmB;AAChC2E,EAAAA,UAAU,GAAG;AACf,CAAC,EAAE;EACD,IAAIC,aAAa,GAAG,CAAC;AAErB,EAAA,MAAMC,UAAU,GAAGH,UAAU,GAAG,IAAI,CAAC;AACrC,EAAA,MAAMI,aAAa,GAAGD,UAAU,GAAGN,aAAa;;AAEhD;AACA,EAAA,MAAMQ,MAAM,GAAG,IAAIC,QAAQ,CAAC;IAC1BC,IAAIA,GAAG,CAAC,CAAC;AAAE;AACXH,IAAAA,aAAa;AACf,GAAC,CAAC;EAEF,gBAAgBI,oBAAoBA,CAACC,cAAc,EAAE;AACnD,IAAA,IAAIC,MAAM,GAAG,EAAE,CAAC;IAChB,IAAIC,aAAa,GAAG,KAAK;IAEzB,IAAI;AACF;AACA,MAAA,WAAW,MAAMC,KAAK,IAAIH,cAAc,EAAE;AACxCC,QAAAA,MAAM,IAAIE,KAAK,CAACC,QAAQ,EAAE,CAAC;;AAE3B;AACA,QAAA,MAAMC,KAAK,GAAGJ,MAAM,CAACpC,KAAK,CAAC,IAAI,CAAC;;AAEhC;AACAoC,QAAAA,MAAM,GAAGI,KAAK,CAACC,GAAG,EAAE;;AAEpB;AACA,QAAA,KAAK,MAAMtC,IAAI,IAAIqC,KAAK,EAAE;AACxB,UAAA,IAAIrC,IAAI,CAACuC,IAAI,EAAE,EAAE;YACf,IAAI;AACF,cAAA,IAAI,CAACf,UAAU,IAAKA,UAAU,IAAI,CAACU,aAAc,EAAE;AACjD,gBAAA,MAAMhC,IAAI,CAACE,KAAK,CAACJ,IAAI,CAAC,CAAC;AACvBkC,gBAAAA,aAAa,GAAG,IAAI;AACtB,cAAA;YACF,CAAC,CAAC,OAAO9D,GAAG,EAAE;AACZ;AACAC,cAAAA,OAAO,CAAC4C,KAAK,CAAC,uBAAuB,EAAE7C,GAAG,CAAC;AAC7C,YAAA;AACF,UAAA;AACF,QAAA;AACF,MAAA;;AAEA;AACA,MAAA,IAAI6D,MAAM,CAACM,IAAI,EAAE,EAAE;QACjB,IAAI;AACF,UAAA,MAAMrC,IAAI,CAACE,KAAK,CAAC6B,MAAM,CAAC;QAC1B,CAAC,CAAC,OAAO7D,GAAG,EAAE;AACZC,UAAAA,OAAO,CAAC4C,KAAK,CAAC,6BAA6B,EAAE7C,GAAG,CAAC;AACnD,QAAA;AACF,MAAA;AACF,IAAA,CAAC,SAAS;AACR;AACA,MAAA,IAAI,CAAC4D,cAAc,CAACQ,SAAS,EAAE;QAC7BR,cAAc,CAACS,OAAO,EAAE;AAC1B,MAAA;AACF,IAAA;AACF,EAAA;EAEA,IAAInD,QAAQ,GAAG,KAAK;EACpB,IAAIoD,aAAa,GAAG,IAAI;;AAExB;AACA,EAAA,CAAC,YAAY;AACX,IAAA,MAAMC,QAAQ,GAAGC,WAAW,CAAC,MAAM;AACjC9B,MAAAA,YAAY,CAAC+B,IAAI,CAAC,eAAe,EAAEpB,aAAa,CAAC;AACjDA,MAAAA,aAAa,GAAG,CAAC;IACnB,CAAC,EAAE,IAAI,CAAC;IAER,IAAI;AACF,MAAA,MAAMH,MAAM,CAACwB,OAAO,CAACC,IAAI,CAAC;AACxBC,QAAAA,WAAW,EAAE5B,aAAa;QAC1BM,UAAU;AACVuB,QAAAA,aAAa,EAAE,IAAI;AACnBC,QAAAA,mBAAmB,EAAE,IAAI;AACzBC,QAAAA,UAAU,EAAEpB,oBAAoB,CAACH,MAAM,CAAC;QACxCwB,UAAUA,CAACnD,GAAG,EAAE;AACdwB,UAAAA,aAAa,EAAE;UACf,OAAO;AACL1D,YAAAA,KAAK,EAAE;AAAEsF,cAAAA,MAAM,EAAElG;AAAgB;WAClC;AACH,QAAA;AACF,OAAC,CAAC;IACJ,CAAC,CAAC,OAAO8D,KAAK,EAAE;AACd5C,MAAAA,OAAO,CAAC4C,KAAK,CAAC,6BAA6B,EAAEA,KAAK,CAAC;AACnD,MAAA,MAAMA,KAAK;AACb,IAAA,CAAC,SAAS;AACR;MACAqC,aAAa,CAACX,QAAQ,CAAC;;AAEvB;AACA,MAAA,IAAID,aAAa,EAAE;AACjBd,QAAAA,MAAM,CAAC2B,cAAc,CAAC,OAAO,EAAEb,aAAa,CAAC;AAC7CA,QAAAA,aAAa,GAAG,IAAI;AACtB,MAAA;;AAEA;MACAd,MAAM,CAAC4B,kBAAkB,EAAE;;AAE3B;AACA,MAAA,IAAI,CAAC5B,MAAM,CAACY,SAAS,EAAE;QACrBZ,MAAM,CAACa,OAAO,EAAE;AAClB,MAAA;;AAEA;AACA3B,MAAAA,YAAY,CAAC+B,IAAI,CAAC,QAAQ,CAAC;MAC3B/B,YAAY,CAAC0C,kBAAkB,EAAE;AACnC,IAAA;AACF,EAAA,CAAC,GAAG;EAEJ,OAAO;IACL9C,GAAG,EAAET,GAAG,IAAI;AACV,MAAA,IAAIX,QAAQ,EAAE;AACZ,QAAA,MAAM,IAAImE,KAAK,CAAC,mDAAmD,CAAC;AACtE,MAAA;AAEA,MAAA,MAAMC,WAAW,GAAG9B,MAAM,CAAC+B,IAAI,CAAC,CAAA,EAAGzD,IAAI,CAACC,SAAS,CAACF,GAAG,CAAC,IAAI,CAAC;MAC3D,IAAI,CAACyD,WAAW,EAAE;AAChB5C,QAAAA,YAAY,CAAC+B,IAAI,CAAC,OAAO,CAAC;;AAE1B;QACAH,aAAa,GAAGA,MAAM;AACpB5B,UAAAA,YAAY,CAAC+B,IAAI,CAAC,QAAQ,CAAC;QAC7B,CAAC;AACDjB,QAAAA,MAAM,CAACgC,IAAI,CAAC,OAAO,EAAElB,aAAa,CAAC;AACrC,MAAA;IACF,CAAC;IACD7B,MAAM,EAAEA,MAAM;AACZvB,MAAAA,QAAQ,GAAG,IAAI;AACfsC,MAAAA,MAAM,CAAC+B,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;AACD7C,IAAAA;GACD;AACH;;ACjJA;AACA,MAAM+C,WAAW,GAAG,IAAIC,WAAW,CAACC,SAAS,CAAC,EAAE,EAAED,WAAW,CAACE,OAAO,CAACC,cAAc,CAAC;AAEtE,SAASC,kBAAkBA,CACxClF,OAAO,EACP/B,eAAe,EACfiC,SAAS,EACToC,MAAM,EACN6C,KAAK,EACLC,UAAU,GAAGtH,mBAAmB,EAChCuH,eAAe,GAAG,KAAK,EACvB;EACA,OAAO,eAAeC,WAAWA,GAAG;IAClC,IAAIC,OAAO,GAAG,CAAC;AACf,IAAA,IAAIC,QAAQ;IACZ,IAAIlF,QAAQ,GAAG,KAAK;IACpB,IAAImF,UAAU,GAAG,KAAK;IACtB,IAAIC,iBAAiB,GAAG,KAAK;IAE7B,eAAeC,oBAAoBA,GAAG;MACpC,IAAI;AACF;AACA,QAAA,MAAMC,QAAQ,GAAG,MAAMtD,MAAM,CAACuD,SAAS,CACrC;AACE9G,UAAAA,KAAK,EAAEd,eAAe;AACtB6H,UAAAA,MAAM,EAAE,GAAG;AACXC,UAAAA,oBAAoB,EAAE,KAAK;AAC3BC,UAAAA,OAAO,EAAE;AACX,SAAC,EACD;AAAEC,UAAAA,UAAU,EAAE;AAAE,SAClB,CAAC;AAED,QAAA,OAAOhH,MAAM,CAACiH,IAAI,CAACN,QAAQ,CAACE,MAAM,CAAC;MACrC,CAAC,CAAC,OAAOnE,CAAC,EAAE;AACVtC,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqC,CAAC,CAAC;AACzB,MAAA;AACF,IAAA;IAEA,SAASwE,MAAMA,CAACL,MAAM,EAAE;MACtB,OAAOxD,MAAM,CAAC6D,MAAM,CAAC;AACnBpH,QAAAA,KAAK,EAAEd,eAAe;AACtBmI,QAAAA,MAAM,EAAE,MAAM;AACdC,QAAAA,IAAI,EAAEjB,UAAU;QAChBD,KAAK;AACL,QAAA,IAAIW,MAAM,GAAG;AAAEQ,UAAAA,OAAO,EAAER;SAAQ,GAAG,EAAE;AACvC,OAAC,CAAC;AACJ,IAAA;IAEA,SAASM,MAAMA,CAACG,EAAE,EAAE;MAClB,OAAOjE,MAAM,CAAC8D,MAAM,CAAC;AACnBI,QAAAA,SAAS,EAAED,EAAE;AACbH,QAAAA,MAAM,EAAE;AACV,OAAC,CAAC;AACJ,IAAA;AAEA,IAAA,IAAIK,cAAc;;AAElB;AACA,IAAA,IAAIpB,eAAe,EAAE;AACnBoB,MAAAA,cAAc,GAAG,MAAMd,oBAAoB,EAAE;AAC/C,IAAA;IAEA,MAAMe,iBAAiB,EAAE;IAEzB,SAASC,UAAUA,CAACC,GAAG,EAAE;AACvBrB,MAAAA,OAAO,IAAI,CAAC;MACZ,IAAI;AACF,QAAA,MAAMtE,GAAG,GAAG,OAAOf,SAAS,KAAK,UAAU,GAAGA,SAAS,CAAC0G,GAAG,CAACN,OAAO,CAAC,GAAGM,GAAG,CAACN,OAAO,CAAC;;AAEnF;AACA,QAAA,IAAI,OAAOrF,GAAG,KAAK,WAAW,EAAE;AAC9B,UAAA;AACF,QAAA;;AAEA;AACA;AACA,QAAA,IAAIK,KAAK,CAACC,OAAO,CAACN,GAAG,CAAC,EAAE;UACtBA,GAAG,CAACO,OAAO,CAACC,CAAC,IAAIzB,OAAO,CAAC0B,GAAG,CAACD,CAAC,CAAC,CAAC;AAChC,UAAA;AACF,QAAA;AAEAzB,QAAAA,OAAO,CAAC0B,GAAG,CAACT,GAAG,CAAC;MAClB,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtC,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqC,CAAC,CAAC;AACzB,MAAA;AACF,IAAA;IAEA,eAAe+E,iBAAiBA,GAAG;AACjCjB,MAAAA,UAAU,GAAG,IAAI;AAEjB,MAAA,MAAMoB,EAAE,GAAGrB,QAAQ,GAAG,MAAMY,MAAM,CAACZ,QAAQ,CAAC,GAAG,MAAMW,MAAM,CAACM,cAAc,CAAC;MAE3E,IAAI,CAACjB,QAAQ,EAAE;AACbX,QAAAA,WAAW,CAACiC,KAAK,CAACD,EAAE,CAACE,IAAI,CAACC,KAAK,CAACC,KAAK,EAAE,CAAC,CAAC;AAC3C,MAAA;MAEAzB,QAAQ,GAAGqB,EAAE,CAACK,UAAU;AACxBzB,MAAAA,UAAU,GAAG,KAAK;MAElB0B,eAAe,CAACN,EAAE,CAAC;AACrB,IAAA;IAEA,eAAeM,eAAeA,CAACvB,QAAQ,EAAE;AACvC;MACAA,QAAQ,CAACmB,IAAI,CAACA,IAAI,CAACvF,OAAO,CAACmF,UAAU,CAAC;AAEtC9B,MAAAA,WAAW,CAACuC,MAAM,CAAC7B,OAAO,CAAC;;AAE3B;MACA,IAAIK,QAAQ,CAACmB,IAAI,CAACC,KAAK,CAACC,KAAK,KAAK1B,OAAO,EAAE;QACzCvF,OAAO,CAAC6B,MAAM,EAAE;AAChB,QAAA;AACF,MAAA;MAEA,IAAI,CAAC6D,iBAAiB,EAAE;QACtB,MAAMgB,iBAAiB,EAAE;AAC3B,MAAA;AACF,IAAA;AAEA1G,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,OAAO,EAAE,YAAY;AAC3C8D,MAAAA,iBAAiB,GAAG,IAAI;AAC1B,IAAA,CAAC,CAAC;AAEF1F,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,QAAQ,EAAE,YAAY;AAC5C8D,MAAAA,iBAAiB,GAAG,KAAK;MAEzB,IAAID,UAAU,IAAInF,QAAQ,EAAE;AAC1B,QAAA;AACF,MAAA;MAEA,MAAMoG,iBAAiB,EAAE;AAC3B,IAAA,CAAC,CAAC;AAEF1G,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,QAAQ,EAAE,MAAM;AACtCtB,MAAAA,QAAQ,GAAG,IAAI;MACfuE,WAAW,CAACwC,IAAI,EAAE;AACpB,IAAA,CAAC,CAAC;EACJ,CAAC;AACH;;AC3Ie,SAASC,mBAAmBA,CAACtH,OAAO,EAAE4C,MAAM,EAAE1C,SAAS,EAAEC,UAAU,EAAE5B,OAAO,EAAE;EAC3F,SAAS6B,UAAUA,GAAG;IACpB,IAAIE,QAAQ,GAAG,KAAK;AAEpB,IAAA,MAAMG,CAAC,GAAGmC,MAAM,CAAChC,IAAI,CAACC,KAAK,CAACV,UAAU,CAAC,CAAC,CAACS,IAAI,CAC3CE,EAAE,CACCC,OAAO,CAACC,IAAI,IAAI;MACf,IAAI;AACF;QACA,IAAIA,IAAI,KAAK,EAAE,EAAE;AACf,UAAA;AACF,QAAA;QAEA,MAAMC,GAAG,GACP,OAAOf,SAAS,KAAK,UAAU,GAAGgB,IAAI,CAACC,SAAS,CAACjB,SAAS,CAACgB,IAAI,CAACE,KAAK,CAACJ,IAAI,CAAC,CAAC,CAAC,GAAGA,IAAI;;AAEtF;AACA,QAAA,IAAI,OAAOC,GAAG,KAAK,WAAW,EAAE;UAC9BR,CAAC,CAACY,MAAM,EAAE;AACV,UAAA;AACF,QAAA;;AAEA;AACA;AACA,QAAA,IAAIC,KAAK,CAACC,OAAO,CAACN,GAAG,CAAC,EAAE;UACtBA,GAAG,CAACO,OAAO,CAACC,CAAC,IAAIzB,OAAO,CAAC0B,GAAG,CAACD,CAAC,CAAC,CAAC;AAChC,UAAA;AACF,QAAA;AAEAzB,QAAAA,OAAO,CAAC0B,GAAG,CAACT,GAAG,CAAC;MAClB,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtC,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqC,CAAC,CAAC;AACzB,MAAA;AACF,IAAA,CAAC,CAAC,CACDC,EAAE,CAAC,OAAO,EAAExC,GAAG,IAAI;AAClBC,MAAAA,OAAO,CAACC,GAAG,CAAC,6BAA6B,EAAEF,GAAG,CAAC;AACjD,IAAA,CAAC,CAAC,CACDwC,EAAE,CAAC,KAAK,EAAE,MAAM;AACf,MAAA,IAAIrD,OAAO,EAAEc,OAAO,CAACC,GAAG,CAAC,qBAAqB,CAAC;MAC/CU,OAAO,CAAC6B,MAAM,EAAE;AAChBvB,MAAAA,QAAQ,GAAG,IAAI;AACjB,IAAA,CAAC,CACL,CAAC;AAEDN,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,OAAO,EAAE,MAAM;AACrC,MAAA,IAAItB,QAAQ,EAAE;MACdG,CAAC,CAACsB,KAAK,EAAE;AACX,IAAA,CAAC,CAAC;AAEF/B,IAAAA,OAAO,CAAC8B,YAAY,CAACF,EAAE,CAAC,QAAQ,EAAE,MAAM;AACtC,MAAA,IAAItB,QAAQ,EAAE;MACdG,CAAC,CAACY,MAAM,EAAE;AACZ,IAAA,CAAC,CAAC;AACJ,EAAA;AAEA,EAAA,OAAO,MAAM;AACXjB,IAAAA,UAAU,EAAE;EACd,CAAC;AACH;;ACnDA;AACA;AACA;AACA,eAAemH,0BAA0BA,CAACC,MAAM,EAAE;EAChD,IAAI;AACF;IACA,MAAMC,UAAU,GAAG,IAAIC,cAAc,CAACC,MAAM,CAACH,MAAM,CAAC;AACpD,IAAA,MAAMI,IAAI,GAAG,MAAMH,UAAU,CAACG,IAAI,EAAE;AACpC,IAAA,MAAMC,OAAO,GAAGD,IAAI,CAACC,OAAO,EAAEC,MAAM;AACpC,IAAA,MAAML,UAAU,CAACM,KAAK,EAAE;AAExB,IAAA,IAAIF,OAAO,EAAE;AACX,MAAA,MAAMG,YAAY,GAAGC,QAAQ,CAACJ,OAAO,CAAChH,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACxD,MAAA,OAAOmH,YAAY;AACrB,IAAA;EACF,CAAC,CAAC,OAAOrG,CAAC,EAAE;AACV;IACA,IAAI;MACF,MAAM8F,UAAU,GAAG,IAAIS,cAAc,CAACP,MAAM,CAACH,MAAM,CAAC;AACpD,MAAA,MAAMI,IAAI,GAAG,MAAMH,UAAU,CAACG,IAAI,EAAE;AACpC,MAAA,MAAMC,OAAO,GAAGD,IAAI,CAACC,OAAO,EAAEC,MAAM;AACpC,MAAA,MAAML,UAAU,CAACM,KAAK,EAAE;AAExB,MAAA,IAAIF,OAAO,EAAE;AACX,QAAA,MAAMG,YAAY,GAAGC,QAAQ,CAACJ,OAAO,CAAChH,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACxD,QAAA,OAAOmH,YAAY;AACrB,MAAA;IACF,CAAC,CAAC,OAAOG,EAAE,EAAE;AACX;AAAA,IAAA;AAEJ,EAAA;;AAEA;AACA,EAAA,OAAO,CAAC;AACV;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,eAAeC,iBAAiBA,CAACC,cAAc,EAAEC,aAAa,EAAEC,YAAY,EAAE;AAC5E;EACA,IAAIF,cAAc,IAAI,OAAOA,cAAc,CAACT,IAAI,KAAK,UAAU,EAAE;AAC/D,IAAA,OAAOS,cAAc;AACvB,EAAA;AAEA,EAAA,MAAMb,MAAM,GAAGa,cAAc,IAAIC,aAAa;;AAE9C;EACA,IAAIC,YAAY,KAAK,CAAC,EAAE;AACtB,IAAA,OAAO,IAAIL,cAAc,CAACP,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA,CAAC,MAAM,IAAIe,YAAY,KAAK,CAAC,EAAE;AAC7B,IAAA,OAAO,IAAIb,cAAc,CAACC,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA;;AAEA;AACA,EAAA,MAAMQ,YAAY,GAAG,MAAMT,0BAA0B,CAACC,MAAM,CAAC;EAE7D,IAAIQ,YAAY,IAAI,CAAC,EAAE;AACrB,IAAA,OAAO,IAAIN,cAAc,CAACC,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA,CAAC,MAAM;AACL,IAAA,OAAO,IAAIU,cAAc,CAACP,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA;AACF;AAEe,eAAegB,WAAWA,CAAC;AACxChK,EAAAA,WAAW,GAAG,KAAK;AACnBR,EAAAA,YAAY,EAAEyK,iBAAiB;AAC/BvK,EAAAA,YAAY,EAAEwK,iBAAiB;EAC/BC,kBAAkB;EAClBC,kBAAkB;EAClBC,mBAAmB;EACnBC,mBAAmB;AACnBvG,EAAAA,UAAU,GAAG1E,mBAAmB;AAChCuH,EAAAA,UAAU,GAAGtH,mBAAmB;EAChC8E,MAAM;EACN3C,QAAQ;AACRE,EAAAA,UAAU,GAAG,IAAI;EACjBlC,eAAe;EACfE,eAAe;EACfC,QAAQ;AACRC,EAAAA,gBAAgB,GAAG,KAAK;EACxBC,4BAA4B;EAC5BG,QAAQ;AACR4G,EAAAA,eAAe,GAAG,KAAK;EACvBF,KAAK;AACL3C,EAAAA,UAAU,GAAG,KAAK;EAClBtC,SAAS;AACT3B,EAAAA,OAAO,GAAG;AACZ,CAAC,EAAE;AACD,EAAA,IAAI,OAAOJ,eAAe,KAAK,WAAW,EAAE;IAC1C,MAAMsG,KAAK,CAAC,oCAAoC,CAAC;AACnD,EAAA;AAEA,EAAA,MAAMsE,mBAAmB,GAAG;AAC1BC,IAAAA,IAAI,EAAEC,OAAO,CAACC,GAAG,CAACC,iBAAiB,IAAI;GACxC;;AAED;AACA,EAAA,MAAMnL,YAAY,GAAG,MAAMoK,iBAAiB,CAC1CK,iBAAiB,IAAIE,kBAAkB,EACvCI,mBAAmB,EACnBF,mBACF,CAAC;AAED,EAAA,MAAM3K,YAAY,GAAG,MAAMkK,iBAAiB,CAC1CM,iBAAiB,IAAIE,kBAAkB,IAAIH,iBAAiB,IAAIE,kBAAkB,EAClFI,mBAAmB,EACnBD,mBACF,CAAC;EAED,MAAMM,aAAa,GAAGrL,oBAAoB,CAAC;IACzCC,YAAY;IACZC,eAAe;IACfC,YAAY;IACZC,eAAe;IACfC,QAAQ;IACRC,gBAAgB;IAChBC,4BAA4B;IAC5BC,OAAO;IACPC,WAAW;AACXC,IAAAA;AACF,GAAC,CAAC;EACF,MAAMuB,OAAO,GAAGqC,iBAAiB,CAAC;IAChCnE,YAAY;IACZC,eAAe;IACfoE,UAAU;IACVC,UAEF,CAAC,CAAC;EAEF,SAAS6G,SAASA,GAAG;IACnB,IAAI,OAAOpJ,QAAQ,KAAK,WAAW,IAAI,OAAOhC,eAAe,KAAK,WAAW,EAAE;MAC7E,MAAMwG,KAAK,CAAC,kEAAkE,CAAC;AACjF,IAAA;AAEA,IAAA,IACG,OAAOxE,QAAQ,KAAK,WAAW,IAAI,OAAOhC,eAAe,KAAK,WAAW,IACzE,OAAOgC,QAAQ,KAAK,WAAW,IAAI,OAAO2C,MAAM,KAAK,WAAY,IACjE,OAAO3E,eAAe,KAAK,WAAW,IAAI,OAAO2E,MAAM,KAAK,WAAY,EACzE;MACA,MAAM6B,KAAK,CAAC,oEAAoE,CAAC;AACnF,IAAA;AAEA,IAAA,IAAI,OAAOxE,QAAQ,KAAK,WAAW,EAAE;MACnC,OAAOF,iBAAiB,CAACC,OAAO,EAAEC,QAAQ,EAAEC,SAAS,EAAEC,UAAU,EAAE5B,OAAO,CAAC;AAC7E,IAAA;AAEA,IAAA,IAAI,OAAON,eAAe,KAAK,WAAW,EAAE;AAC1C,MAAA,OAAOiH,kBAAkB,CACvBlF,OAAO,EACP/B,eAAe,EACfiC,SAAS,EACTlC,YAAY,EACZmH,KAAK,EACLC,UAAU,EACVC,eACF,CAAC;AACH,IAAA;AAEA,IAAA,IAAI,OAAOzC,MAAM,KAAK,WAAW,EAAE;MACjC,OAAO0E,mBAAmB,CAACtH,OAAO,EAAE4C,MAAM,EAAE1C,SAAS,EAAEC,UAAU,EAAE5B,OAAO,CAAC;AAC7E,IAAA;AAEA,IAAA,OAAO,IAAI;AACb,EAAA;AAEA,EAAA,MAAM+K,MAAM,GAAGD,SAAS,EAAE;EAE1B,IAAI;IACF,MAAM7J,WAAW,GAAG,MAAMtB,YAAY,CAACW,OAAO,CAACY,MAAM,CAAC;AAAEV,MAAAA,KAAK,EAAEZ;AAAgB,KAAC,CAAC;IAEjF,IAAIqB,WAAW,KAAK,KAAK,EAAE;MACzB,MAAM4J,aAAa,EAAE;AACrBE,MAAAA,MAAM,EAAE;AACV,IAAA,CAAC,MAAM,IAAI9K,WAAW,KAAK,IAAI,EAAE;AAC/B,MAAA,MAAMN,YAAY,CAACW,OAAO,CAACa,MAAM,CAAC;AAAEX,QAAAA,KAAK,EAAEZ;AAAgB,OAAC,CAAC;MAC7D,MAAMiL,aAAa,EAAE;AACrBE,MAAAA,MAAM,EAAE;AACV,IAAA,CAAC,MAAM;AACLA,MAAAA,MAAM,EAAE;AACV,IAAA;EACF,CAAC,CAAC,OAAOrH,KAAK,EAAE;AACd5C,IAAAA,OAAO,CAAC4C,KAAK,CAAC,iCAAiC,EAAEA,KAAK,CAAC;AACzD,EAAA,CAAC,SAAS;AACR;AAAA,EAAA;EAGF,OAAO;IAAEsH,MAAM,EAAEvJ,OAAO,CAAC8B;GAAc;AACzC;;;;"}
1	+ {"version":3,"file":"node-es-transformer.esm.js","sources":["../src/_constants.js","../src/_create-mapping.js","../src/_csv-parser-options.js","../src/_file-reader.js","../src/_index-queue.js","../src/_index-reader.js","../src/_infer-mappings.js","../src/_stream-reader.js","../src/main.js"],"sourcesContent":["// In earlier versions this was used to set the number of docs to index in a\n// single bulk request. Since we switched to use the helpers.bulk() method from\n// the ES client, this now translates to the `flushBytes` option of the helper.\n// However, for kind of a backwards compability with the old values, this uses\n// KBytes instead of Bytes. It will be multiplied by 1024 in the index queue.\nexport const DEFAULT_BUFFER_SIZE = 5120;\n\n// The default number of docs to fetch in a single search request when reindexing.\nexport const DEFAULT_SEARCH_SIZE = 1000;\n","export default function createMappingFactory({\n sourceClient,\n sourceIndexName,\n targetClient,\n targetIndexName,\n mappings,\n inferredIngestPipeline,\n mappingsOverride,\n indexMappingTotalFieldsLimit,\n verbose,\n deleteIndex,\n pipeline,\n}) {\n return async () => {\n let targetMappings = mappingsOverride ? undefined : mappings;\n let defaultPipeline = pipeline;\n\n if (sourceClient && sourceIndexName && typeof targetMappings === 'undefined') {\n try {\n const mapping = await sourceClient.indices.getMapping({\n index: sourceIndexName,\n });\n if (mapping[sourceIndexName]) {\n targetMappings = mapping[sourceIndexName].mappings;\n } else {\n const allMappings = Object.values(mapping);\n if (allMappings.length > 0) {\n targetMappings = Object.values(mapping)[0].mappings;\n }\n }\n } catch (err) {\n console.log('Error reading source mapping', err);\n return;\n }\n }\n\n if (typeof targetMappings === 'object' && targetMappings !== null) {\n if (mappingsOverride) {\n targetMappings = {\n ...targetMappings,\n properties: {\n ...targetMappings.properties,\n ...mappings,\n },\n };\n }\n\n try {\n const indexExists = await targetClient.indices.exists({ index: targetIndexName });\n\n if (indexExists === true && deleteIndex === true) {\n await targetClient.indices.delete({ index: targetIndexName });\n }\n\n if (indexExists === false \|\| deleteIndex === true) {\n if (\n typeof defaultPipeline === 'undefined' &&\n typeof inferredIngestPipeline === 'object' &&\n inferredIngestPipeline !== null &&\n typeof targetClient?.ingest?.putPipeline === 'function'\n ) {\n const inferredPipelineName = `${targetIndexName}-inferred-pipeline`;\n\n try {\n await targetClient.ingest.putPipeline({\n id: inferredPipelineName,\n ...inferredIngestPipeline,\n });\n defaultPipeline = inferredPipelineName;\n if (verbose) console.log(`Created inferred ingest pipeline ${inferredPipelineName}`);\n } catch (err) {\n console.log('Error creating inferred ingest pipeline', err);\n }\n }\n\n const settings = {\n ...(defaultPipeline !== undefined\n ? {\n 'index.default_pipeline': defaultPipeline,\n }\n : {}),\n ...(indexMappingTotalFieldsLimit !== undefined\n ? {\n 'index.mapping.total_fields.limit': indexMappingTotalFieldsLimit,\n 'index.number_of_shards': 1,\n 'index.number_of_replicas': 0,\n }\n : {}),\n };\n\n const resp = await targetClient.indices.create({\n index: targetIndexName,\n mappings: targetMappings,\n ...(Object.keys(settings).length > 0 ? { settings } : {}),\n });\n if (verbose) console.log('Created target mapping', resp);\n }\n } catch (err) {\n console.log('Error creating target mapping', err);\n }\n }\n };\n}\n","export default function getCsvParserOptions(csvOptions = {}, skipHeader = false) {\n const options = {\n bom: true,\n columns: true,\n trim: true,\n skip_empty_lines: true,\n ...csvOptions,\n };\n\n const consumesHeader = options.columns === true \|\| typeof options.columns === 'function';\n\n if (skipHeader && !consumesHeader && typeof options.from_line === 'undefined') {\n options.from_line = 2;\n }\n\n return options;\n}\n","import fs from 'fs';\nimport { parse } from 'csv-parse';\nimport es from 'event-stream';\nimport { globSync } from 'glob';\nimport split from 'split2';\n\nimport getCsvParserOptions from './_csv-parser-options';\n\nexport default function fileReaderFactory(\n indexer,\n fileName,\n transform,\n splitRegex,\n verbose,\n skipHeader = false,\n sourceFormat = 'ndjson',\n csvOptions = {},\n) {\n function addParsedDoc(parsed, file, streamRef) {\n const context = { fileName: file };\n const doc = typeof transform === 'function' ? transform(parsed, context) : parsed;\n\n // if doc is null/undefined we'll skip indexing it\n if (doc === null \|\| typeof doc === 'undefined') {\n streamRef.resume();\n return;\n }\n\n // the transform callback may return an array of docs so we can emit\n // multiple docs from a single line\n if (Array.isArray(doc)) {\n doc.forEach(d => {\n if (d === null \|\| typeof d === 'undefined') return;\n indexer.add(d);\n });\n return;\n }\n\n indexer.add(doc);\n }\n\n function createNdjsonReader(file) {\n let skippedHeader = false;\n\n const s = fs\n .createReadStream(file)\n .pipe(split(splitRegex))\n .pipe(\n es\n .mapSync(line => {\n try {\n // skip empty lines\n if (line === '') {\n return;\n }\n\n if (skipHeader && !skippedHeader) {\n skippedHeader = true;\n return;\n }\n\n const parsed = JSON.parse(line);\n addParsedDoc(parsed, file, s);\n } catch (e) {\n console.log('error', e);\n }\n })\n .on('error', err => {\n console.log('Error while reading file.', err);\n }),\n );\n\n return s;\n }\n\n function createCsvReader(file) {\n const parserOptions = getCsvParserOptions(csvOptions, skipHeader);\n\n const s = fs\n .createReadStream(file)\n .pipe(parse(parserOptions))\n .pipe(\n es\n .mapSync(record => {\n try {\n addParsedDoc(record, file, s);\n } catch (e) {\n console.log('error', e);\n }\n })\n .on('error', err => {\n console.log('Error while reading CSV file.', err);\n }),\n );\n\n return s;\n }\n\n function startIndex(files) {\n let finished = false;\n\n if (files.length === 0) {\n indexer.finish();\n return;\n }\n\n const file = files.shift();\n\n const s = sourceFormat === 'csv' ? createCsvReader(file) : createNdjsonReader(file);\n\n s.on('end', () => {\n if (verbose) console.log('Read entire file: ', file);\n if (files.length > 0) {\n startIndex(files);\n return;\n }\n\n indexer.finish();\n finished = true;\n });\n\n indexer.queueEmitter.on('pause', () => {\n if (finished) return;\n s.pause();\n });\n\n indexer.queueEmitter.on('resume', () => {\n if (finished) return;\n s.resume();\n });\n }\n\n return () => {\n try {\n const files = globSync(fileName);\n startIndex(files);\n } catch (error) {\n console.log('Error matching files:', error);\n }\n };\n}\n","import { PassThrough } from 'stream';\n\nimport { DEFAULT_BUFFER_SIZE } from './_constants';\n\nconst EventEmitter = require('events');\n\nconst parallelCalls = 5;\n\n// a simple helper queue to bulk index documents\nexport default function indexQueueFactory({\n targetClient: client,\n targetIndexName,\n bufferSize = DEFAULT_BUFFER_SIZE,\n}) {\n const queueEmitter = new EventEmitter();\n let docsPerSecond = 0;\n\n const flushBytes = bufferSize * 1024; // Convert KB to Bytes\n const highWaterMark = flushBytes * parallelCalls;\n\n // Create a PassThrough stream (readable + writable) for proper backpressure\n const stream = new PassThrough({\n highWaterMark, // Buffer size for backpressure management\n });\n\n async function* ndjsonStreamIterator(readableStream) {\n let buffer = ''; // To hold the incomplete data\n\n try {\n // Iterate over the stream using async iteration\n for await (const chunk of readableStream) {\n buffer += chunk.toString(); // Accumulate the chunk data in the buffer\n\n // Split the buffer into lines (NDJSON items)\n const lines = buffer.split('\\n');\n\n // The last line might be incomplete, so hold it back in the buffer\n buffer = lines.pop();\n\n // Yield each complete JSON object\n for (const line of lines) {\n if (!line.trim()) {\n continue;\n }\n\n try {\n yield JSON.parse(line); // Parse and yield the JSON object\n } catch (err) {\n // Handle JSON parse errors if necessary\n console.error('Failed to parse JSON:', err);\n }\n }\n }\n\n // Handle any remaining data in the buffer after the stream ends\n if (buffer.trim()) {\n try {\n yield JSON.parse(buffer);\n } catch (err) {\n console.error('Failed to parse final JSON:', err);\n }\n }\n } finally {\n // Ensure the stream is properly cleaned up if the iterator is terminated early\n if (!readableStream.destroyed) {\n readableStream.destroy();\n }\n }\n }\n\n let finished = false;\n let drainListener = null;\n\n // Async IIFE to start bulk indexing\n (async () => {\n const interval = setInterval(() => {\n queueEmitter.emit('docsPerSecond', docsPerSecond);\n docsPerSecond = 0;\n }, 1000);\n\n try {\n await client.helpers.bulk({\n concurrency: parallelCalls,\n flushBytes,\n flushInterval: 1000,\n refreshOnCompletion: true,\n datasource: ndjsonStreamIterator(stream),\n onDocument(doc) {\n docsPerSecond++;\n return {\n index: { _index: targetIndexName },\n };\n },\n });\n } catch (error) {\n console.error('Error during bulk indexing:', error);\n throw error;\n } finally {\n // Clean up interval\n clearInterval(interval);\n\n // Remove drain listener if it exists\n if (drainListener) {\n stream.removeListener('drain', drainListener);\n drainListener = null;\n }\n\n // Remove all listeners from stream\n stream.removeAllListeners();\n\n // Properly destroy the stream to prevent open handles\n if (!stream.destroyed) {\n stream.destroy();\n }\n\n // Emit finish and clean up queue emitter listeners\n queueEmitter.emit('finish');\n queueEmitter.removeAllListeners();\n }\n })();\n\n return {\n add: doc => {\n if (finished) {\n throw new Error('Unexpected doc added after indexer should finish.');\n }\n\n const canContinue = stream.write(`${JSON.stringify(doc)}\\n`);\n if (!canContinue) {\n queueEmitter.emit('pause');\n\n // Store the listener so we can clean it up later\n drainListener = () => {\n queueEmitter.emit('resume');\n };\n stream.once('drain', drainListener);\n }\n },\n finish: () => {\n finished = true;\n stream.end();\n },\n queueEmitter,\n };\n}\n","import cliProgress from 'cli-progress';\n\nimport { DEFAULT_SEARCH_SIZE } from './_constants';\n\n// create a new progress bar instance and use shades_classic theme\nconst progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\nexport default function indexReaderFactory(\n indexer,\n sourceIndexName,\n transform,\n client,\n query,\n searchSize = DEFAULT_SEARCH_SIZE,\n populatedFields = false,\n) {\n return async function indexReader() {\n let docsNum = 0;\n let scrollId;\n let finished = false;\n let readActive = false;\n let backPressurePause = false;\n\n async function fetchPopulatedFields() {\n try {\n // Get all populated fields from the index\n const response = await client.fieldCaps(\n {\n index: sourceIndexName,\n fields: '',\n include_empty_fields: false,\n filters: '-metadata',\n },\n { maxRetries: 0 },\n );\n\n return Object.keys(response.fields);\n } catch (e) {\n console.log('error', e);\n }\n }\n\n function search(fields) {\n return client.search({\n index: sourceIndexName,\n scroll: '600s',\n size: searchSize,\n query,\n ...(fields ? { _source: fields } : {}),\n });\n }\n\n function scroll(id) {\n return client.scroll({\n scroll_id: id,\n scroll: '600s',\n });\n }\n\n let fieldsWithData;\n\n // identify populated fields\n if (populatedFields) {\n fieldsWithData = await fetchPopulatedFields();\n }\n\n await fetchNextResponse();\n\n function processHit(hit) {\n docsNum += 1;\n try {\n const doc = typeof transform === 'function' ? transform(hit._source) : hit._source; // eslint-disable-line no-underscore-dangle\n\n // if doc is undefined we'll skip indexing it\n if (typeof doc === 'undefined') {\n return;\n }\n\n // the transform callback may return an array of docs so we can emit\n // multiple docs from a single line\n if (Array.isArray(doc)) {\n doc.forEach(d => indexer.add(d));\n return;\n }\n\n indexer.add(doc);\n } catch (e) {\n console.log('error', e);\n }\n }\n\n async function fetchNextResponse() {\n readActive = true;\n\n const sc = scrollId ? await scroll(scrollId) : await search(fieldsWithData);\n\n if (!scrollId) {\n progressBar.start(sc.hits.total.value, 0);\n }\n\n scrollId = sc._scroll_id;\n readActive = false;\n\n processResponse(sc);\n }\n\n async function processResponse(response) {\n // collect the docs from this response\n response.hits.hits.forEach(processHit);\n\n progressBar.update(docsNum);\n\n // check to see if we have collected all of the docs\n if (response.hits.total.value === docsNum) {\n indexer.finish();\n return;\n }\n\n if (!backPressurePause) {\n await fetchNextResponse();\n }\n }\n\n indexer.queueEmitter.on('pause', async () => {\n backPressurePause = true;\n });\n\n indexer.queueEmitter.on('resume', async () => {\n backPressurePause = false;\n\n if (readActive \|\| finished) {\n return;\n }\n\n await fetchNextResponse();\n });\n\n indexer.queueEmitter.on('finish', () => {\n finished = true;\n progressBar.stop();\n });\n };\n}\n","import fs from 'fs';\nimport { globSync } from 'glob';\n\nconst DEFAULT_INFER_MAPPINGS_SAMPLE_BYTES = 100000;\nconst DEFAULT_INFER_MAPPINGS_LINES_TO_SAMPLE = 1000;\n\nfunction readSample(filePath, sampleBytes) {\n const fd = fs.openSync(filePath, 'r');\n\n try {\n const buffer = Buffer.alloc(sampleBytes);\n const bytesRead = fs.readSync(fd, buffer, 0, sampleBytes, 0);\n return buffer.subarray(0, bytesRead).toString('utf8');\n } finally {\n fs.closeSync(fd);\n }\n}\n\nfunction emptyInferenceResult(mappings) {\n return {\n mappings,\n ingestPipeline: undefined,\n };\n}\n\nexport default async function inferMappingsFromSource({\n targetClient,\n fileName,\n sourceFormat,\n csvOptions,\n skipHeader,\n mappings,\n inferMappings,\n inferMappingsOptions,\n verbose,\n}) {\n if (!inferMappings \|\| typeof mappings !== 'undefined' \|\| typeof fileName === 'undefined') {\n return emptyInferenceResult(mappings);\n }\n\n if (\n typeof targetClient?.textStructure?.findStructure !== 'function' \|\|\n sourceFormat === 'xml' \|\|\n sourceFormat === 'semi_structured_text'\n ) {\n return emptyInferenceResult(mappings);\n }\n\n const files = globSync(fileName);\n\n if (files.length === 0) {\n if (verbose) console.log(`No files matched for mapping inference: ${fileName}`);\n return emptyInferenceResult(mappings);\n }\n\n const { sampleBytes = DEFAULT_INFER_MAPPINGS_SAMPLE_BYTES, ...requestParams } =\n inferMappingsOptions \|\| {};\n\n const sampleText = readSample(files[0], sampleBytes);\n\n if (!sampleText \|\| sampleText.trim() === '') {\n if (verbose) console.log('Skipping mapping inference because the sample text is empty.');\n return emptyInferenceResult(mappings);\n }\n\n const params = {\n body: sampleText,\n lines_to_sample: DEFAULT_INFER_MAPPINGS_LINES_TO_SAMPLE,\n ...requestParams,\n };\n\n if (typeof params.format === 'undefined') {\n params.format = sourceFormat === 'csv' ? 'delimited' : 'ndjson';\n }\n\n if (sourceFormat === 'csv') {\n if (typeof params.delimiter === 'undefined' && typeof csvOptions?.delimiter === 'string') {\n params.delimiter = csvOptions.delimiter;\n }\n\n if (typeof params.quote === 'undefined' && typeof csvOptions?.quote === 'string') {\n params.quote = csvOptions.quote;\n }\n\n if (typeof params.has_header_row === 'undefined' && typeof csvOptions?.columns === 'boolean') {\n params.has_header_row = csvOptions.columns;\n }\n\n if (typeof params.has_header_row === 'undefined' && skipHeader) {\n params.has_header_row = true;\n }\n }\n\n try {\n const response = await targetClient.textStructure.findStructure(params);\n\n if (response?.mappings && verbose) {\n console.log(`Inferred mappings via _text_structure/find_structure from ${files[0]}`);\n }\n\n if (response?.ingest_pipeline && verbose) {\n console.log('Inferred ingest pipeline via _text_structure/find_structure');\n }\n\n return {\n mappings: response?.mappings \|\| mappings,\n ingestPipeline: response?.ingest_pipeline,\n };\n } catch (error) {\n if (verbose) {\n console.log('Could not infer mappings via _text_structure/find_structure:', error.message);\n }\n\n return emptyInferenceResult(mappings);\n }\n}\n","import { parse } from 'csv-parse';\nimport es from 'event-stream';\nimport split from 'split2';\n\nimport getCsvParserOptions from './_csv-parser-options';\n\nexport default function streamReaderFactory(\n indexer,\n stream,\n transform,\n splitRegex,\n verbose,\n skipHeader = false,\n sourceFormat = 'ndjson',\n csvOptions = {},\n) {\n function addParsedDoc(parsed, streamRef) {\n const doc = typeof transform === 'function' ? transform(parsed) : parsed;\n\n // if doc is null/undefined we'll skip indexing it\n if (doc === null \|\| typeof doc === 'undefined') {\n streamRef.resume();\n return;\n }\n\n // the transform callback may return an array of docs so we can emit\n // multiple docs from a single line\n if (Array.isArray(doc)) {\n doc.forEach(d => {\n if (d === null \|\| typeof d === 'undefined') return;\n indexer.add(d);\n });\n return;\n }\n\n indexer.add(doc);\n }\n\n function startIndex() {\n let finished = false;\n\n const s =\n sourceFormat === 'csv'\n ? stream.pipe(parse(getCsvParserOptions(csvOptions, skipHeader))).pipe(\n es\n .mapSync(record => {\n try {\n addParsedDoc(record, s);\n } catch (e) {\n console.log('error', e);\n }\n })\n .on('error', err => {\n console.log('Error while reading CSV stream.', err);\n }),\n )\n : (() => {\n let skippedHeader = false;\n\n return stream.pipe(split(splitRegex)).pipe(\n es\n .mapSync(line => {\n try {\n // skip empty lines\n if (line === '') {\n return;\n }\n\n if (skipHeader && !skippedHeader) {\n skippedHeader = true;\n return;\n }\n\n const parsed = JSON.parse(line);\n addParsedDoc(parsed, s);\n } catch (e) {\n console.log('error', e);\n }\n })\n .on('error', err => {\n console.log('Error while reading stream.', err);\n }),\n );\n })();\n\n s.on('end', () => {\n if (verbose) console.log('Read entire stream.');\n indexer.finish();\n finished = true;\n });\n\n indexer.queueEmitter.on('pause', () => {\n if (finished) return;\n s.pause();\n });\n\n indexer.queueEmitter.on('resume', () => {\n if (finished) return;\n s.resume();\n });\n }\n\n return () => {\n startIndex();\n };\n}\n","import elasticsearch9 from 'es9';\nimport elasticsearch8 from 'es8';\n\nimport { DEFAULT_BUFFER_SIZE, DEFAULT_SEARCH_SIZE } from './_constants';\nimport createMappingFactory from './_create-mapping';\nimport fileReaderFactory from './_file-reader';\nimport indexQueueFactory from './_index-queue';\nimport indexReaderFactory from './_index-reader';\nimport inferMappingsFromSource from './_infer-mappings';\nimport streamReaderFactory from './_stream-reader';\n\n/\n Detect Elasticsearch version by querying the cluster\n /\nasync function detectElasticsearchVersion(config) {\n try {\n // Try with v9 client first (most common for new setups)\n const testClient = new elasticsearch9.Client(config);\n const info = await testClient.info();\n const version = info.version?.number;\n await testClient.close();\n\n if (version) {\n const majorVersion = parseInt(version.split('.')[0], 10);\n return majorVersion;\n }\n } catch (e) {\n // If v9 client fails, try v8 client\n try {\n const testClient = new elasticsearch8.Client(config);\n const info = await testClient.info();\n const version = info.version?.number;\n await testClient.close();\n\n if (version) {\n const majorVersion = parseInt(version.split('.')[0], 10);\n return majorVersion;\n }\n } catch (e2) {\n // Could not detect version\n }\n }\n\n // Default to v9 if detection fails\n return 9;\n}\n\n/\n Create or validate an Elasticsearch client\n * @param {Object\|Client} clientOrConfig - Either a client instance or config object\n * @param {Object} defaultConfig - Default configuration to use if creating a new client\n * @param {number} [forceVersion] - Force a specific ES client version (8 or 9)\n */\nasync function getOrCreateClient(clientOrConfig, defaultConfig, forceVersion) {\n // If already a client instance, return it\n if (clientOrConfig && typeof clientOrConfig.info === 'function') {\n return clientOrConfig;\n }\n\n const config = clientOrConfig \|\| defaultConfig;\n\n // If version is forced, use the specified client\n if (forceVersion === 8) {\n return new elasticsearch8.Client(config);\n } else if (forceVersion === 9) {\n return new elasticsearch9.Client(config);\n }\n\n // Auto-detect version\n const majorVersion = await detectElasticsearchVersion(config);\n\n if (majorVersion >= 9) {\n return new elasticsearch9.Client(config);\n } else {\n return new elasticsearch8.Client(config);\n }\n}\n\nexport default async function transformer({\n deleteIndex = false,\n sourceClient: sourceClientInput,\n targetClient: targetClientInput,\n sourceClientConfig,\n targetClientConfig,\n sourceClientVersion,\n targetClientVersion,\n bufferSize = DEFAULT_BUFFER_SIZE,\n searchSize = DEFAULT_SEARCH_SIZE,\n stream,\n fileName,\n sourceFormat = 'ndjson',\n csvOptions = {},\n splitRegex = /\\n/,\n sourceIndexName,\n targetIndexName,\n mappings,\n mappingsOverride = false,\n inferMappings = false,\n inferMappingsOptions = {},\n indexMappingTotalFieldsLimit,\n pipeline,\n populatedFields = false,\n query,\n skipHeader = false,\n transform,\n verbose = true,\n}) {\n if (typeof targetIndexName === 'undefined') {\n throw Error('targetIndexName must be specified.');\n }\n\n const defaultClientConfig = {\n node: process.env.ELASTICSEARCH_URL \|\| 'http://localhost:9200',\n };\n\n // Support both old (config) and new (client instance) patterns\n const sourceClient = await getOrCreateClient(\n sourceClientInput \|\| sourceClientConfig,\n defaultClientConfig,\n sourceClientVersion,\n );\n\n const targetClient = await getOrCreateClient(\n targetClientInput \|\| targetClientConfig \|\| sourceClientInput \|\| sourceClientConfig,\n defaultClientConfig,\n targetClientVersion,\n );\n\n const inferenceResult = await inferMappingsFromSource({\n targetClient,\n fileName,\n sourceFormat,\n csvOptions,\n skipHeader,\n mappings,\n inferMappings,\n inferMappingsOptions,\n verbose,\n });\n\n const createMapping = createMappingFactory({\n sourceClient,\n sourceIndexName,\n targetClient,\n targetIndexName,\n mappings: inferenceResult.mappings,\n inferredIngestPipeline: inferenceResult.ingestPipeline,\n mappingsOverride,\n indexMappingTotalFieldsLimit,\n verbose,\n deleteIndex,\n pipeline,\n });\n const indexer = indexQueueFactory({\n targetClient,\n targetIndexName,\n bufferSize,\n verbose,\n });\n\n function validateSourceFormat() {\n if (sourceFormat !== 'ndjson' && sourceFormat !== 'csv') {\n throw Error(`Unsupported sourceFormat: ${sourceFormat}. Use \"ndjson\" or \"csv\".`);\n }\n }\n\n function getReader() {\n if (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') {\n throw Error('Only either one of fileName or sourceIndexName can be specified.');\n }\n\n if (\n (typeof fileName !== 'undefined' && typeof sourceIndexName !== 'undefined') \|\|\n (typeof fileName !== 'undefined' && typeof stream !== 'undefined') \|\|\n (typeof sourceIndexName !== 'undefined' && typeof stream !== 'undefined')\n ) {\n throw Error('Only one of fileName, sourceIndexName, or stream can be specified.');\n }\n\n if (typeof fileName !== 'undefined') {\n validateSourceFormat();\n return fileReaderFactory(\n indexer,\n fileName,\n transform,\n splitRegex,\n verbose,\n skipHeader,\n sourceFormat,\n csvOptions,\n );\n }\n\n if (typeof sourceIndexName !== 'undefined') {\n return indexReaderFactory(\n indexer,\n sourceIndexName,\n transform,\n sourceClient,\n query,\n searchSize,\n populatedFields,\n );\n }\n\n if (typeof stream !== 'undefined') {\n validateSourceFormat();\n return streamReaderFactory(\n indexer,\n stream,\n transform,\n splitRegex,\n verbose,\n skipHeader,\n sourceFormat,\n csvOptions,\n );\n }\n\n return null;\n }\n\n const reader = getReader();\n\n try {\n const indexExists = await targetClient.indices.exists({ index: targetIndexName });\n\n if (indexExists === false) {\n await createMapping();\n reader();\n } else if (deleteIndex === true) {\n await targetClient.indices.delete({ index: targetIndexName });\n await createMapping();\n reader();\n } else {\n reader();\n }\n } catch (error) {\n console.error('Error checking index existence:', error);\n } finally {\n // targetClient.close();\n }\n\n return { events: indexer.queueEmitter };\n}\n"],"names":["DEFAULT_BUFFER_SIZE","DEFAULT_SEARCH_SIZE","createMappingFactory","sourceClient","sourceIndexName","targetClient","targetIndexName","mappings","inferredIngestPipeline","mappingsOverride","indexMappingTotalFieldsLimit","verbose","deleteIndex","pipeline","targetMappings","undefined","defaultPipeline","mapping","indices","getMapping","index","allMappings","Object","values","length","err","console","log","properties","indexExists","exists","delete","ingest","putPipeline","inferredPipelineName","id","settings","resp","create","keys","getCsvParserOptions","csvOptions","skipHeader","options","bom","columns","trim","skip_empty_lines","consumesHeader","from_line","fileReaderFactory","indexer","fileName","transform","splitRegex","sourceFormat","addParsedDoc","parsed","file","streamRef","context","doc","resume","Array","isArray","forEach","d","add","createNdjsonReader","skippedHeader","s","fs","createReadStream","pipe","split","es","mapSync","line","JSON","parse","e","on","createCsvReader","parserOptions","record","startIndex","files","finished","finish","shift","queueEmitter","pause","globSync","error","EventEmitter","require","parallelCalls","indexQueueFactory","client","bufferSize","docsPerSecond","flushBytes","highWaterMark","stream","PassThrough","ndjsonStreamIterator","readableStream","buffer","chunk","toString","lines","pop","destroyed","destroy","drainListener","interval","setInterval","emit","helpers","bulk","concurrency","flushInterval","refreshOnCompletion","datasource","onDocument","_index","clearInterval","removeListener","removeAllListeners","Error","canContinue","write","stringify","once","end","progressBar","cliProgress","SingleBar","Presets","shades_classic","indexReaderFactory","query","searchSize","populatedFields","indexReader","docsNum","scrollId","readActive","backPressurePause","fetchPopulatedFields","response","fieldCaps","fields","include_empty_fields","filters","maxRetries","search","scroll","size","_source","scroll_id","fieldsWithData","fetchNextResponse","processHit","hit","sc","start","hits","total","value","_scroll_id","processResponse","update","stop","DEFAULT_INFER_MAPPINGS_SAMPLE_BYTES","DEFAULT_INFER_MAPPINGS_LINES_TO_SAMPLE","readSample","filePath","sampleBytes","fd","openSync","Buffer","alloc","bytesRead","readSync","subarray","closeSync","emptyInferenceResult","ingestPipeline","inferMappingsFromSource","inferMappings","inferMappingsOptions","textStructure","findStructure","requestParams","sampleText","params","body","lines_to_sample","format","delimiter","quote","has_header_row","ingest_pipeline","message","streamReaderFactory","detectElasticsearchVersion","config","testClient","elasticsearch9","Client","info","version","number","close","majorVersion","parseInt","elasticsearch8","e2","getOrCreateClient","clientOrConfig","defaultConfig","forceVersion","transformer","sourceClientInput","targetClientInput","sourceClientConfig","targetClientConfig","sourceClientVersion","targetClientVersion","defaultClientConfig","node","process","env","ELASTICSEARCH_URL","inferenceResult","createMapping","validateSourceFormat","getReader","reader","events"],"mappings":";;;;;;;;;;AAAA;AACA;AACA;AACA;AACA;AACO,MAAMA,mBAAmB,GAAG,IAAI;;AAEvC;AACO,MAAMC,mBAAmB,GAAG,IAAI;;ACRxB,SAASC,oBAAoBA,CAAC;EAC3CC,YAAY;EACZC,eAAe;EACfC,YAAY;EACZC,eAAe;EACfC,QAAQ;EACRC,sBAAsB;EACtBC,gBAAgB;EAChBC,4BAA4B;EAC5BC,OAAO;EACPC,WAAW;AACXC,EAAAA;AACF,CAAC,EAAE;AACD,EAAA,OAAO,YAAY;AACjB,IAAA,IAAIC,cAAc,GAAGL,gBAAgB,GAAGM,SAAS,GAAGR,QAAQ;IAC5D,IAAIS,eAAe,GAAGH,QAAQ;IAE9B,IAAIV,YAAY,IAAIC,eAAe,IAAI,OAAOU,cAAc,KAAK,WAAW,EAAE;MAC5E,IAAI;QACF,MAAMG,OAAO,GAAG,MAAMd,YAAY,CAACe,OAAO,CAACC,UAAU,CAAC;AACpDC,UAAAA,KAAK,EAAEhB;AACT,SAAC,CAAC;AACF,QAAA,IAAIa,OAAO,CAACb,eAAe,CAAC,EAAE;AAC5BU,UAAAA,cAAc,GAAGG,OAAO,CAACb,eAAe,CAAC,CAACG,QAAQ;AACpD,QAAA,CAAC,MAAM;AACL,UAAA,MAAMc,WAAW,GAAGC,MAAM,CAACC,MAAM,CAACN,OAAO,CAAC;AAC1C,UAAA,IAAII,WAAW,CAACG,MAAM,GAAG,CAAC,EAAE;YAC1BV,cAAc,GAAGQ,MAAM,CAACC,MAAM,CAACN,OAAO,CAAC,CAAC,CAAC,CAAC,CAACV,QAAQ;AACrD,UAAA;AACF,QAAA;MACF,CAAC,CAAC,OAAOkB,GAAG,EAAE;AACZC,QAAAA,OAAO,CAACC,GAAG,CAAC,8BAA8B,EAAEF,GAAG,CAAC;AAChD,QAAA;AACF,MAAA;AACF,IAAA;IAEA,IAAI,OAAOX,cAAc,KAAK,QAAQ,IAAIA,cAAc,KAAK,IAAI,EAAE;AACjE,MAAA,IAAIL,gBAAgB,EAAE;AACpBK,QAAAA,cAAc,GAAG;AACf,UAAA,GAAGA,cAAc;AACjBc,UAAAA,UAAU,EAAE;YACV,GAAGd,cAAc,CAACc,UAAU;YAC5B,GAAGrB;AACL;SACD;AACH,MAAA;MAEA,IAAI;QACF,MAAMsB,WAAW,GAAG,MAAMxB,YAAY,CAACa,OAAO,CAACY,MAAM,CAAC;AAAEV,UAAAA,KAAK,EAAEd;AAAgB,SAAC,CAAC;AAEjF,QAAA,IAAIuB,WAAW,KAAK,IAAI,IAAIjB,WAAW,KAAK,IAAI,EAAE;AAChD,UAAA,MAAMP,YAAY,CAACa,OAAO,CAACa,MAAM,CAAC;AAAEX,YAAAA,KAAK,EAAEd;AAAgB,WAAC,CAAC;AAC/D,QAAA;AAEA,QAAA,IAAIuB,WAAW,KAAK,KAAK,IAAIjB,WAAW,KAAK,IAAI,EAAE;UACjD,IACE,OAAOI,eAAe,KAAK,WAAW,IACtC,OAAOR,sBAAsB,KAAK,QAAQ,IAC1CA,sBAAsB,KAAK,IAAI,IAC/B,OAAOH,YAAY,EAAE2B,MAAM,EAAEC,WAAW,KAAK,UAAU,EACvD;AACA,YAAA,MAAMC,oBAAoB,GAAG,CAAA,EAAG5B,eAAe,CAAA,kBAAA,CAAoB;YAEnE,IAAI;AACF,cAAA,MAAMD,YAAY,CAAC2B,MAAM,CAACC,WAAW,CAAC;AACpCE,gBAAAA,EAAE,EAAED,oBAAoB;gBACxB,GAAG1B;AACL,eAAC,CAAC;AACFQ,cAAAA,eAAe,GAAGkB,oBAAoB;cACtC,IAAIvB,OAAO,EAAEe,OAAO,CAACC,GAAG,CAAC,CAAA,iCAAA,EAAoCO,oBAAoB,CAAA,CAAE,CAAC;YACtF,CAAC,CAAC,OAAOT,GAAG,EAAE;AACZC,cAAAA,OAAO,CAACC,GAAG,CAAC,yCAAyC,EAAEF,GAAG,CAAC;AAC7D,YAAA;AACF,UAAA;AAEA,UAAA,MAAMW,QAAQ,GAAG;YACf,IAAIpB,eAAe,KAAKD,SAAS,GAC7B;AACE,cAAA,wBAAwB,EAAEC;aAC3B,GACD,EAAE,CAAC;YACP,IAAIN,4BAA4B,KAAKK,SAAS,GAC1C;AACE,cAAA,kCAAkC,EAAEL,4BAA4B;AAChE,cAAA,wBAAwB,EAAE,CAAC;AAC3B,cAAA,0BAA0B,EAAE;aAC7B,GACD,EAAE;WACP;UAED,MAAM2B,IAAI,GAAG,MAAMhC,YAAY,CAACa,OAAO,CAACoB,MAAM,CAAC;AAC7ClB,YAAAA,KAAK,EAAEd,eAAe;AACtBC,YAAAA,QAAQ,EAAEO,cAAc;YACxB,IAAIQ,MAAM,CAACiB,IAAI,CAACH,QAAQ,CAAC,CAACZ,MAAM,GAAG,CAAC,GAAG;AAAEY,cAAAA;aAAU,GAAG,EAAE;AAC1D,WAAC,CAAC;UACF,IAAIzB,OAAO,EAAEe,OAAO,CAACC,GAAG,CAAC,wBAAwB,EAAEU,IAAI,CAAC;AAC1D,QAAA;MACF,CAAC,CAAC,OAAOZ,GAAG,EAAE;AACZC,QAAAA,OAAO,CAACC,GAAG,CAAC,+BAA+B,EAAEF,GAAG,CAAC;AACnD,MAAA;AACF,IAAA;EACF,CAAC;AACH;;ACtGe,SAASe,mBAAmBA,CAACC,UAAU,GAAG,EAAE,EAAEC,UAAU,GAAG,KAAK,EAAE;AAC/E,EAAA,MAAMC,OAAO,GAAG;AACdC,IAAAA,GAAG,EAAE,IAAI;AACTC,IAAAA,OAAO,EAAE,IAAI;AACbC,IAAAA,IAAI,EAAE,IAAI;AACVC,IAAAA,gBAAgB,EAAE,IAAI;IACtB,GAAGN;GACJ;AAED,EAAA,MAAMO,cAAc,GAAGL,OAAO,CAACE,OAAO,KAAK,IAAI,IAAI,OAAOF,OAAO,CAACE,OAAO,KAAK,UAAU;EAExF,IAAIH,UAAU,IAAI,CAACM,cAAc,IAAI,OAAOL,OAAO,CAACM,SAAS,KAAK,WAAW,EAAE;IAC7EN,OAAO,CAACM,SAAS,GAAG,CAAC;AACvB,EAAA;AAEA,EAAA,OAAON,OAAO;AAChB;;ACRe,SAASO,iBAAiBA,CACvCC,OAAO,EACPC,QAAQ,EACRC,SAAS,EACTC,UAAU,EACV3C,OAAO,EACP+B,UAAU,GAAG,KAAK,EAClBa,YAAY,GAAG,QAAQ,EACvBd,UAAU,GAAG,EAAE,EACf;AACA,EAAA,SAASe,YAAYA,CAACC,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE;AAC7C,IAAA,MAAMC,OAAO,GAAG;AAAER,MAAAA,QAAQ,EAAEM;KAAM;AAClC,IAAA,MAAMG,GAAG,GAAG,OAAOR,SAAS,KAAK,UAAU,GAAGA,SAAS,CAACI,MAAM,EAAEG,OAAO,CAAC,GAAGH,MAAM;;AAEjF;IACA,IAAII,GAAG,KAAK,IAAI,IAAI,OAAOA,GAAG,KAAK,WAAW,EAAE;MAC9CF,SAAS,CAACG,MAAM,EAAE;AAClB,MAAA;AACF,IAAA;;AAEA;AACA;AACA,IAAA,IAAIC,KAAK,CAACC,OAAO,CAACH,GAAG,CAAC,EAAE;AACtBA,MAAAA,GAAG,CAACI,OAAO,CAACC,CAAC,IAAI;QACf,IAAIA,CAAC,KAAK,IAAI,IAAI,OAAOA,CAAC,KAAK,WAAW,EAAE;AAC5Cf,QAAAA,OAAO,CAACgB,GAAG,CAACD,CAAC,CAAC;AAChB,MAAA,CAAC,CAAC;AACF,MAAA;AACF,IAAA;AAEAf,IAAAA,OAAO,CAACgB,GAAG,CAACN,GAAG,CAAC;AAClB,EAAA;EAEA,SAASO,kBAAkBA,CAACV,IAAI,EAAE;IAChC,IAAIW,aAAa,GAAG,KAAK;IAEzB,MAAMC,CAAC,GAAGC,EAAE,CACTC,gBAAgB,CAACd,IAAI,CAAC,CACtBe,IAAI,CAACC,KAAK,CAACpB,UAAU,CAAC,CAAC,CACvBmB,IAAI,CACHE,EAAE,CACCC,OAAO,CAACC,IAAI,IAAI;MACf,IAAI;AACF;QACA,IAAIA,IAAI,KAAK,EAAE,EAAE;AACf,UAAA;AACF,QAAA;AAEA,QAAA,IAAInC,UAAU,IAAI,CAAC2B,aAAa,EAAE;AAChCA,UAAAA,aAAa,GAAG,IAAI;AACpB,UAAA;AACF,QAAA;AAEA,QAAA,MAAMZ,MAAM,GAAGqB,IAAI,CAACC,KAAK,CAACF,IAAI,CAAC;AAC/BrB,QAAAA,YAAY,CAACC,MAAM,EAAEC,IAAI,EAAEY,CAAC,CAAC;MAC/B,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtD,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqD,CAAC,CAAC;AACzB,MAAA;AACF,IAAA,CAAC,CAAC,CACDC,EAAE,CAAC,OAAO,EAAExD,GAAG,IAAI;AAClBC,MAAAA,OAAO,CAACC,GAAG,CAAC,2BAA2B,EAAEF,GAAG,CAAC;AAC/C,IAAA,CAAC,CACL,CAAC;AAEH,IAAA,OAAO6C,CAAC;AACV,EAAA;EAEA,SAASY,eAAeA,CAACxB,IAAI,EAAE;AAC7B,IAAA,MAAMyB,aAAa,GAAG3C,mBAAmB,CAACC,UAAU,EAAEC,UAAU,CAAC;IAEjE,MAAM4B,CAAC,GAAGC,EAAE,CACTC,gBAAgB,CAACd,IAAI,CAAC,CACtBe,IAAI,CAACM,KAAK,CAACI,aAAa,CAAC,CAAC,CAC1BV,IAAI,CACHE,EAAE,CACCC,OAAO,CAACQ,MAAM,IAAI;MACjB,IAAI;AACF5B,QAAAA,YAAY,CAAC4B,MAAM,EAAE1B,IAAI,EAAEY,CAAC,CAAC;MAC/B,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtD,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqD,CAAC,CAAC;AACzB,MAAA;AACF,IAAA,CAAC,CAAC,CACDC,EAAE,CAAC,OAAO,EAAExD,GAAG,IAAI;AAClBC,MAAAA,OAAO,CAACC,GAAG,CAAC,+BAA+B,EAAEF,GAAG,CAAC;AACnD,IAAA,CAAC,CACL,CAAC;AAEH,IAAA,OAAO6C,CAAC;AACV,EAAA;EAEA,SAASe,UAAUA,CAACC,KAAK,EAAE;IACzB,IAAIC,QAAQ,GAAG,KAAK;AAEpB,IAAA,IAAID,KAAK,CAAC9D,MAAM,KAAK,CAAC,EAAE;MACtB2B,OAAO,CAACqC,MAAM,EAAE;AAChB,MAAA;AACF,IAAA;AAEA,IAAA,MAAM9B,IAAI,GAAG4B,KAAK,CAACG,KAAK,EAAE;AAE1B,IAAA,MAAMnB,CAAC,GAAGf,YAAY,KAAK,KAAK,GAAG2B,eAAe,CAACxB,IAAI,CAAC,GAAGU,kBAAkB,CAACV,IAAI,CAAC;AAEnFY,IAAAA,CAAC,CAACW,EAAE,CAAC,KAAK,EAAE,MAAM;MAChB,IAAItE,OAAO,EAAEe,OAAO,CAACC,GAAG,CAAC,oBAAoB,EAAE+B,IAAI,CAAC;AACpD,MAAA,IAAI4B,KAAK,CAAC9D,MAAM,GAAG,CAAC,EAAE;QACpB6D,UAAU,CAACC,KAAK,CAAC;AACjB,QAAA;AACF,MAAA;MAEAnC,OAAO,CAACqC,MAAM,EAAE;AAChBD,MAAAA,QAAQ,GAAG,IAAI;AACjB,IAAA,CAAC,CAAC;AAEFpC,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,OAAO,EAAE,MAAM;AACrC,MAAA,IAAIM,QAAQ,EAAE;MACdjB,CAAC,CAACqB,KAAK,EAAE;AACX,IAAA,CAAC,CAAC;AAEFxC,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,QAAQ,EAAE,MAAM;AACtC,MAAA,IAAIM,QAAQ,EAAE;MACdjB,CAAC,CAACR,MAAM,EAAE;AACZ,IAAA,CAAC,CAAC;AACJ,EAAA;AAEA,EAAA,OAAO,MAAM;IACX,IAAI;AACF,MAAA,MAAMwB,KAAK,GAAGM,QAAQ,CAACxC,QAAQ,CAAC;MAChCiC,UAAU,CAACC,KAAK,CAAC;IACnB,CAAC,CAAC,OAAOO,KAAK,EAAE;AACdnE,MAAAA,OAAO,CAACC,GAAG,CAAC,uBAAuB,EAAEkE,KAAK,CAAC;AAC7C,IAAA;EACF,CAAC;AACH;;ACxIA,MAAMC,YAAY,GAAGC,OAAO,CAAC,QAAQ,CAAC;AAEtC,MAAMC,aAAa,GAAG,CAAC;;AAEvB;AACe,SAASC,iBAAiBA,CAAC;AACxC5F,EAAAA,YAAY,EAAE6F,MAAM;EACpB5F,eAAe;AACf6F,EAAAA,UAAU,GAAGnG;AACf,CAAC,EAAE;AACD,EAAA,MAAM0F,YAAY,GAAG,IAAII,YAAY,EAAE;EACvC,IAAIM,aAAa,GAAG,CAAC;AAErB,EAAA,MAAMC,UAAU,GAAGF,UAAU,GAAG,IAAI,CAAC;AACrC,EAAA,MAAMG,aAAa,GAAGD,UAAU,GAAGL,aAAa;;AAEhD;AACA,EAAA,MAAMO,MAAM,GAAG,IAAIC,WAAW,CAAC;AAC7BF,IAAAA,aAAa;AACf,GAAC,CAAC;EAEF,gBAAgBG,oBAAoBA,CAACC,cAAc,EAAE;AACnD,IAAA,IAAIC,MAAM,GAAG,EAAE,CAAC;;IAEhB,IAAI;AACF;AACA,MAAA,WAAW,MAAMC,KAAK,IAAIF,cAAc,EAAE;AACxCC,QAAAA,MAAM,IAAIC,KAAK,CAACC,QAAQ,EAAE,CAAC;;AAE3B;AACA,QAAA,MAAMC,KAAK,GAAGH,MAAM,CAACjC,KAAK,CAAC,IAAI,CAAC;;AAEhC;AACAiC,QAAAA,MAAM,GAAGG,KAAK,CAACC,GAAG,EAAE;;AAEpB;AACA,QAAA,KAAK,MAAMlC,IAAI,IAAIiC,KAAK,EAAE;AACxB,UAAA,IAAI,CAACjC,IAAI,CAAC/B,IAAI,EAAE,EAAE;AAChB,YAAA;AACF,UAAA;UAEA,IAAI;AACF,YAAA,MAAMgC,IAAI,CAACC,KAAK,CAACF,IAAI,CAAC,CAAC;UACzB,CAAC,CAAC,OAAOpD,GAAG,EAAE;AACZ;AACAC,YAAAA,OAAO,CAACmE,KAAK,CAAC,uBAAuB,EAAEpE,GAAG,CAAC;AAC7C,UAAA;AACF,QAAA;AACF,MAAA;;AAEA;AACA,MAAA,IAAIkF,MAAM,CAAC7D,IAAI,EAAE,EAAE;QACjB,IAAI;AACF,UAAA,MAAMgC,IAAI,CAACC,KAAK,CAAC4B,MAAM,CAAC;QAC1B,CAAC,CAAC,OAAOlF,GAAG,EAAE;AACZC,UAAAA,OAAO,CAACmE,KAAK,CAAC,6BAA6B,EAAEpE,GAAG,CAAC;AACnD,QAAA;AACF,MAAA;AACF,IAAA,CAAC,SAAS;AACR;AACA,MAAA,IAAI,CAACiF,cAAc,CAACM,SAAS,EAAE;QAC7BN,cAAc,CAACO,OAAO,EAAE;AAC1B,MAAA;AACF,IAAA;AACF,EAAA;EAEA,IAAI1B,QAAQ,GAAG,KAAK;EACpB,IAAI2B,aAAa,GAAG,IAAI;;AAExB;AACA,EAAA,CAAC,YAAY;AACX,IAAA,MAAMC,QAAQ,GAAGC,WAAW,CAAC,MAAM;AACjC1B,MAAAA,YAAY,CAAC2B,IAAI,CAAC,eAAe,EAAEjB,aAAa,CAAC;AACjDA,MAAAA,aAAa,GAAG,CAAC;IACnB,CAAC,EAAE,IAAI,CAAC;IAER,IAAI;AACF,MAAA,MAAMF,MAAM,CAACoB,OAAO,CAACC,IAAI,CAAC;AACxBC,QAAAA,WAAW,EAAExB,aAAa;QAC1BK,UAAU;AACVoB,QAAAA,aAAa,EAAE,IAAI;AACnBC,QAAAA,mBAAmB,EAAE,IAAI;AACzBC,QAAAA,UAAU,EAAElB,oBAAoB,CAACF,MAAM,CAAC;QACxCqB,UAAUA,CAAC/D,GAAG,EAAE;AACduC,UAAAA,aAAa,EAAE;UACf,OAAO;AACLhF,YAAAA,KAAK,EAAE;AAAEyG,cAAAA,MAAM,EAAEvH;AAAgB;WAClC;AACH,QAAA;AACF,OAAC,CAAC;IACJ,CAAC,CAAC,OAAOuF,KAAK,EAAE;AACdnE,MAAAA,OAAO,CAACmE,KAAK,CAAC,6BAA6B,EAAEA,KAAK,CAAC;AACnD,MAAA,MAAMA,KAAK;AACb,IAAA,CAAC,SAAS;AACR;MACAiC,aAAa,CAACX,QAAQ,CAAC;;AAEvB;AACA,MAAA,IAAID,aAAa,EAAE;AACjBX,QAAAA,MAAM,CAACwB,cAAc,CAAC,OAAO,EAAEb,aAAa,CAAC;AAC7CA,QAAAA,aAAa,GAAG,IAAI;AACtB,MAAA;;AAEA;MACAX,MAAM,CAACyB,kBAAkB,EAAE;;AAE3B;AACA,MAAA,IAAI,CAACzB,MAAM,CAACS,SAAS,EAAE;QACrBT,MAAM,CAACU,OAAO,EAAE;AAClB,MAAA;;AAEA;AACAvB,MAAAA,YAAY,CAAC2B,IAAI,CAAC,QAAQ,CAAC;MAC3B3B,YAAY,CAACsC,kBAAkB,EAAE;AACnC,IAAA;AACF,EAAA,CAAC,GAAG;EAEJ,OAAO;IACL7D,GAAG,EAAEN,GAAG,IAAI;AACV,MAAA,IAAI0B,QAAQ,EAAE;AACZ,QAAA,MAAM,IAAI0C,KAAK,CAAC,mDAAmD,CAAC;AACtE,MAAA;AAEA,MAAA,MAAMC,WAAW,GAAG3B,MAAM,CAAC4B,KAAK,CAAC,CAAA,EAAGrD,IAAI,CAACsD,SAAS,CAACvE,GAAG,CAAC,IAAI,CAAC;MAC5D,IAAI,CAACqE,WAAW,EAAE;AAChBxC,QAAAA,YAAY,CAAC2B,IAAI,CAAC,OAAO,CAAC;;AAE1B;QACAH,aAAa,GAAGA,MAAM;AACpBxB,UAAAA,YAAY,CAAC2B,IAAI,CAAC,QAAQ,CAAC;QAC7B,CAAC;AACDd,QAAAA,MAAM,CAAC8B,IAAI,CAAC,OAAO,EAAEnB,aAAa,CAAC;AACrC,MAAA;IACF,CAAC;IACD1B,MAAM,EAAEA,MAAM;AACZD,MAAAA,QAAQ,GAAG,IAAI;MACfgB,MAAM,CAAC+B,GAAG,EAAE;IACd,CAAC;AACD5C,IAAAA;GACD;AACH;;AC5IA;AACA,MAAM6C,WAAW,GAAG,IAAIC,WAAW,CAACC,SAAS,CAAC,EAAE,EAAED,WAAW,CAACE,OAAO,CAACC,cAAc,CAAC;AAEtE,SAASC,kBAAkBA,CACxCzF,OAAO,EACP/C,eAAe,EACfiD,SAAS,EACT6C,MAAM,EACN2C,KAAK,EACLC,UAAU,GAAG7I,mBAAmB,EAChC8I,eAAe,GAAG,KAAK,EACvB;EACA,OAAO,eAAeC,WAAWA,GAAG;IAClC,IAAIC,OAAO,GAAG,CAAC;AACf,IAAA,IAAIC,QAAQ;IACZ,IAAI3D,QAAQ,GAAG,KAAK;IACpB,IAAI4D,UAAU,GAAG,KAAK;IACtB,IAAIC,iBAAiB,GAAG,KAAK;IAE7B,eAAeC,oBAAoBA,GAAG;MACpC,IAAI;AACF;AACA,QAAA,MAAMC,QAAQ,GAAG,MAAMpD,MAAM,CAACqD,SAAS,CACrC;AACEnI,UAAAA,KAAK,EAAEhB,eAAe;AACtBoJ,UAAAA,MAAM,EAAE,GAAG;AACXC,UAAAA,oBAAoB,EAAE,KAAK;AAC3BC,UAAAA,OAAO,EAAE;AACX,SAAC,EACD;AAAEC,UAAAA,UAAU,EAAE;AAAE,SAClB,CAAC;AAED,QAAA,OAAOrI,MAAM,CAACiB,IAAI,CAAC+G,QAAQ,CAACE,MAAM,CAAC;MACrC,CAAC,CAAC,OAAOxE,CAAC,EAAE;AACVtD,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqD,CAAC,CAAC;AACzB,MAAA;AACF,IAAA;IAEA,SAAS4E,MAAMA,CAACJ,MAAM,EAAE;MACtB,OAAOtD,MAAM,CAAC0D,MAAM,CAAC;AACnBxI,QAAAA,KAAK,EAAEhB,eAAe;AACtByJ,QAAAA,MAAM,EAAE,MAAM;AACdC,QAAAA,IAAI,EAAEhB,UAAU;QAChBD,KAAK;AACL,QAAA,IAAIW,MAAM,GAAG;AAAEO,UAAAA,OAAO,EAAEP;SAAQ,GAAG,EAAE;AACvC,OAAC,CAAC;AACJ,IAAA;IAEA,SAASK,MAAMA,CAAC1H,EAAE,EAAE;MAClB,OAAO+D,MAAM,CAAC2D,MAAM,CAAC;AACnBG,QAAAA,SAAS,EAAE7H,EAAE;AACb0H,QAAAA,MAAM,EAAE;AACV,OAAC,CAAC;AACJ,IAAA;AAEA,IAAA,IAAII,cAAc;;AAElB;AACA,IAAA,IAAIlB,eAAe,EAAE;AACnBkB,MAAAA,cAAc,GAAG,MAAMZ,oBAAoB,EAAE;AAC/C,IAAA;IAEA,MAAMa,iBAAiB,EAAE;IAEzB,SAASC,UAAUA,CAACC,GAAG,EAAE;AACvBnB,MAAAA,OAAO,IAAI,CAAC;MACZ,IAAI;AACF,QAAA,MAAMpF,GAAG,GAAG,OAAOR,SAAS,KAAK,UAAU,GAAGA,SAAS,CAAC+G,GAAG,CAACL,OAAO,CAAC,GAAGK,GAAG,CAACL,OAAO,CAAC;;AAEnF;AACA,QAAA,IAAI,OAAOlG,GAAG,KAAK,WAAW,EAAE;AAC9B,UAAA;AACF,QAAA;;AAEA;AACA;AACA,QAAA,IAAIE,KAAK,CAACC,OAAO,CAACH,GAAG,CAAC,EAAE;UACtBA,GAAG,CAACI,OAAO,CAACC,CAAC,IAAIf,OAAO,CAACgB,GAAG,CAACD,CAAC,CAAC,CAAC;AAChC,UAAA;AACF,QAAA;AAEAf,QAAAA,OAAO,CAACgB,GAAG,CAACN,GAAG,CAAC;MAClB,CAAC,CAAC,OAAOmB,CAAC,EAAE;AACVtD,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqD,CAAC,CAAC;AACzB,MAAA;AACF,IAAA;IAEA,eAAekF,iBAAiBA,GAAG;AACjCf,MAAAA,UAAU,GAAG,IAAI;AAEjB,MAAA,MAAMkB,EAAE,GAAGnB,QAAQ,GAAG,MAAMW,MAAM,CAACX,QAAQ,CAAC,GAAG,MAAMU,MAAM,CAACK,cAAc,CAAC;MAE3E,IAAI,CAACf,QAAQ,EAAE;AACbX,QAAAA,WAAW,CAAC+B,KAAK,CAACD,EAAE,CAACE,IAAI,CAACC,KAAK,CAACC,KAAK,EAAE,CAAC,CAAC;AAC3C,MAAA;MAEAvB,QAAQ,GAAGmB,EAAE,CAACK,UAAU;AACxBvB,MAAAA,UAAU,GAAG,KAAK;MAElBwB,eAAe,CAACN,EAAE,CAAC;AACrB,IAAA;IAEA,eAAeM,eAAeA,CAACrB,QAAQ,EAAE;AACvC;MACAA,QAAQ,CAACiB,IAAI,CAACA,IAAI,CAACtG,OAAO,CAACkG,UAAU,CAAC;AAEtC5B,MAAAA,WAAW,CAACqC,MAAM,CAAC3B,OAAO,CAAC;;AAE3B;MACA,IAAIK,QAAQ,CAACiB,IAAI,CAACC,KAAK,CAACC,KAAK,KAAKxB,OAAO,EAAE;QACzC9F,OAAO,CAACqC,MAAM,EAAE;AAChB,QAAA;AACF,MAAA;MAEA,IAAI,CAAC4D,iBAAiB,EAAE;QACtB,MAAMc,iBAAiB,EAAE;AAC3B,MAAA;AACF,IAAA;AAEA/G,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,OAAO,EAAE,YAAY;AAC3CmE,MAAAA,iBAAiB,GAAG,IAAI;AAC1B,IAAA,CAAC,CAAC;AAEFjG,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,QAAQ,EAAE,YAAY;AAC5CmE,MAAAA,iBAAiB,GAAG,KAAK;MAEzB,IAAID,UAAU,IAAI5D,QAAQ,EAAE;AAC1B,QAAA;AACF,MAAA;MAEA,MAAM2E,iBAAiB,EAAE;AAC3B,IAAA,CAAC,CAAC;AAEF/G,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,QAAQ,EAAE,MAAM;AACtCM,MAAAA,QAAQ,GAAG,IAAI;MACfgD,WAAW,CAACsC,IAAI,EAAE;AACpB,IAAA,CAAC,CAAC;EACJ,CAAC;AACH;;AC3IA,MAAMC,mCAAmC,GAAG,MAAM;AAClD,MAAMC,sCAAsC,GAAG,IAAI;AAEnD,SAASC,UAAUA,CAACC,QAAQ,EAAEC,WAAW,EAAE;EACzC,MAAMC,EAAE,GAAG5G,EAAE,CAAC6G,QAAQ,CAACH,QAAQ,EAAE,GAAG,CAAC;EAErC,IAAI;AACF,IAAA,MAAMtE,MAAM,GAAG0E,MAAM,CAACC,KAAK,CAACJ,WAAW,CAAC;AACxC,IAAA,MAAMK,SAAS,GAAGhH,EAAE,CAACiH,QAAQ,CAACL,EAAE,EAAExE,MAAM,EAAE,CAAC,EAAEuE,WAAW,EAAE,CAAC,CAAC;AAC5D,IAAA,OAAOvE,MAAM,CAAC8E,QAAQ,CAAC,CAAC,EAAEF,SAAS,CAAC,CAAC1E,QAAQ,CAAC,MAAM,CAAC;AACvD,EAAA,CAAC,SAAS;AACRtC,IAAAA,EAAE,CAACmH,SAAS,CAACP,EAAE,CAAC;AAClB,EAAA;AACF;AAEA,SAASQ,oBAAoBA,CAACpL,QAAQ,EAAE;EACtC,OAAO;IACLA,QAAQ;AACRqL,IAAAA,cAAc,EAAE7K;GACjB;AACH;AAEe,eAAe8K,uBAAuBA,CAAC;EACpDxL,YAAY;EACZ+C,QAAQ;EACRG,YAAY;EACZd,UAAU;EACVC,UAAU;EACVnC,QAAQ;EACRuL,aAAa;EACbC,oBAAoB;AACpBpL,EAAAA;AACF,CAAC,EAAE;AACD,EAAA,IAAI,CAACmL,aAAa,IAAI,OAAOvL,QAAQ,KAAK,WAAW,IAAI,OAAO6C,QAAQ,KAAK,WAAW,EAAE;IACxF,OAAOuI,oBAAoB,CAACpL,QAAQ,CAAC;AACvC,EAAA;AAEA,EAAA,IACE,OAAOF,YAAY,EAAE2L,aAAa,EAAEC,aAAa,KAAK,UAAU,IAChE1I,YAAY,KAAK,KAAK,IACtBA,YAAY,KAAK,sBAAsB,EACvC;IACA,OAAOoI,oBAAoB,CAACpL,QAAQ,CAAC;AACvC,EAAA;AAEA,EAAA,MAAM+E,KAAK,GAAGM,QAAQ,CAACxC,QAAQ,CAAC;AAEhC,EAAA,IAAIkC,KAAK,CAAC9D,MAAM,KAAK,CAAC,EAAE;IACtB,IAAIb,OAAO,EAAEe,OAAO,CAACC,GAAG,CAAC,CAAA,wCAAA,EAA2CyB,QAAQ,CAAA,CAAE,CAAC;IAC/E,OAAOuI,oBAAoB,CAACpL,QAAQ,CAAC;AACvC,EAAA;EAEA,MAAM;AAAE2K,IAAAA,WAAW,GAAGJ,mCAAmC;IAAE,GAAGoB;AAAc,GAAC,GAC3EH,oBAAoB,IAAI,EAAE;EAE5B,MAAMI,UAAU,GAAGnB,UAAU,CAAC1F,KAAK,CAAC,CAAC,CAAC,EAAE4F,WAAW,CAAC;EAEpD,IAAI,CAACiB,UAAU,IAAIA,UAAU,CAACrJ,IAAI,EAAE,KAAK,EAAE,EAAE;AAC3C,IAAA,IAAInC,OAAO,EAAEe,OAAO,CAACC,GAAG,CAAC,8DAA8D,CAAC;IACxF,OAAOgK,oBAAoB,CAACpL,QAAQ,CAAC;AACvC,EAAA;AAEA,EAAA,MAAM6L,MAAM,GAAG;AACbC,IAAAA,IAAI,EAAEF,UAAU;AAChBG,IAAAA,eAAe,EAAEvB,sCAAsC;IACvD,GAAGmB;GACJ;AAED,EAAA,IAAI,OAAOE,MAAM,CAACG,MAAM,KAAK,WAAW,EAAE;IACxCH,MAAM,CAACG,MAAM,GAAGhJ,YAAY,KAAK,KAAK,GAAG,WAAW,GAAG,QAAQ;AACjE,EAAA;EAEA,IAAIA,YAAY,KAAK,KAAK,EAAE;AAC1B,IAAA,IAAI,OAAO6I,MAAM,CAACI,SAAS,KAAK,WAAW,IAAI,OAAO/J,UAAU,EAAE+J,SAAS,KAAK,QAAQ,EAAE;AACxFJ,MAAAA,MAAM,CAACI,SAAS,GAAG/J,UAAU,CAAC+J,SAAS;AACzC,IAAA;AAEA,IAAA,IAAI,OAAOJ,MAAM,CAACK,KAAK,KAAK,WAAW,IAAI,OAAOhK,UAAU,EAAEgK,KAAK,KAAK,QAAQ,EAAE;AAChFL,MAAAA,MAAM,CAACK,KAAK,GAAGhK,UAAU,CAACgK,KAAK;AACjC,IAAA;AAEA,IAAA,IAAI,OAAOL,MAAM,CAACM,cAAc,KAAK,WAAW,IAAI,OAAOjK,UAAU,EAAEI,OAAO,KAAK,SAAS,EAAE;AAC5FuJ,MAAAA,MAAM,CAACM,cAAc,GAAGjK,UAAU,CAACI,OAAO;AAC5C,IAAA;IAEA,IAAI,OAAOuJ,MAAM,CAACM,cAAc,KAAK,WAAW,IAAIhK,UAAU,EAAE;MAC9D0J,MAAM,CAACM,cAAc,GAAG,IAAI;AAC9B,IAAA;AACF,EAAA;EAEA,IAAI;IACF,MAAMpD,QAAQ,GAAG,MAAMjJ,YAAY,CAAC2L,aAAa,CAACC,aAAa,CAACG,MAAM,CAAC;AAEvE,IAAA,IAAI9C,QAAQ,EAAE/I,QAAQ,IAAII,OAAO,EAAE;MACjCe,OAAO,CAACC,GAAG,CAAC,CAAA,0DAAA,EAA6D2D,KAAK,CAAC,CAAC,CAAC,CAAA,CAAE,CAAC;AACtF,IAAA;AAEA,IAAA,IAAIgE,QAAQ,EAAEqD,eAAe,IAAIhM,OAAO,EAAE;AACxCe,MAAAA,OAAO,CAACC,GAAG,CAAC,6DAA6D,CAAC;AAC5E,IAAA;IAEA,OAAO;AACLpB,MAAAA,QAAQ,EAAE+I,QAAQ,EAAE/I,QAAQ,IAAIA,QAAQ;MACxCqL,cAAc,EAAEtC,QAAQ,EAAEqD;KAC3B;EACH,CAAC,CAAC,OAAO9G,KAAK,EAAE;AACd,IAAA,IAAIlF,OAAO,EAAE;MACXe,OAAO,CAACC,GAAG,CAAC,8DAA8D,EAAEkE,KAAK,CAAC+G,OAAO,CAAC;AAC5F,IAAA;IAEA,OAAOjB,oBAAoB,CAACpL,QAAQ,CAAC;AACvC,EAAA;AACF;;AC7Ge,SAASsM,mBAAmBA,CACzC1J,OAAO,EACPoD,MAAM,EACNlD,SAAS,EACTC,UAAU,EACV3C,OAAO,EACP+B,UAAU,GAAG,KAAK,EAClBa,YAAY,GAAG,QAAQ,EACvBd,UAAU,GAAG,EAAE,EACf;AACA,EAAA,SAASe,YAAYA,CAACC,MAAM,EAAEE,SAAS,EAAE;AACvC,IAAA,MAAME,GAAG,GAAG,OAAOR,SAAS,KAAK,UAAU,GAAGA,SAAS,CAACI,MAAM,CAAC,GAAGA,MAAM;;AAExE;IACA,IAAII,GAAG,KAAK,IAAI,IAAI,OAAOA,GAAG,KAAK,WAAW,EAAE;MAC9CF,SAAS,CAACG,MAAM,EAAE;AAClB,MAAA;AACF,IAAA;;AAEA;AACA;AACA,IAAA,IAAIC,KAAK,CAACC,OAAO,CAACH,GAAG,CAAC,EAAE;AACtBA,MAAAA,GAAG,CAACI,OAAO,CAACC,CAAC,IAAI;QACf,IAAIA,CAAC,KAAK,IAAI,IAAI,OAAOA,CAAC,KAAK,WAAW,EAAE;AAC5Cf,QAAAA,OAAO,CAACgB,GAAG,CAACD,CAAC,CAAC;AAChB,MAAA,CAAC,CAAC;AACF,MAAA;AACF,IAAA;AAEAf,IAAAA,OAAO,CAACgB,GAAG,CAACN,GAAG,CAAC;AAClB,EAAA;EAEA,SAASwB,UAAUA,GAAG;IACpB,IAAIE,QAAQ,GAAG,KAAK;AAEpB,IAAA,MAAMjB,CAAC,GACLf,YAAY,KAAK,KAAK,GAClBgD,MAAM,CAAC9B,IAAI,CAACM,KAAK,CAACvC,mBAAmB,CAACC,UAAU,EAAEC,UAAU,CAAC,CAAC,CAAC,CAAC+B,IAAI,CAClEE,EAAE,CACCC,OAAO,CAACQ,MAAM,IAAI;MACjB,IAAI;AACF5B,QAAAA,YAAY,CAAC4B,MAAM,EAAEd,CAAC,CAAC;MACzB,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtD,QAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqD,CAAC,CAAC;AACzB,MAAA;AACF,IAAA,CAAC,CAAC,CACDC,EAAE,CAAC,OAAO,EAAExD,GAAG,IAAI;AAClBC,MAAAA,OAAO,CAACC,GAAG,CAAC,iCAAiC,EAAEF,GAAG,CAAC;AACrD,IAAA,CAAC,CACL,CAAC,GACD,CAAC,MAAM;MACL,IAAI4C,aAAa,GAAG,KAAK;AAEzB,MAAA,OAAOkC,MAAM,CAAC9B,IAAI,CAACC,KAAK,CAACpB,UAAU,CAAC,CAAC,CAACmB,IAAI,CACxCE,EAAE,CACCC,OAAO,CAACC,IAAI,IAAI;QACf,IAAI;AACF;UACA,IAAIA,IAAI,KAAK,EAAE,EAAE;AACf,YAAA;AACF,UAAA;AAEA,UAAA,IAAInC,UAAU,IAAI,CAAC2B,aAAa,EAAE;AAChCA,YAAAA,aAAa,GAAG,IAAI;AACpB,YAAA;AACF,UAAA;AAEA,UAAA,MAAMZ,MAAM,GAAGqB,IAAI,CAACC,KAAK,CAACF,IAAI,CAAC;AAC/BrB,UAAAA,YAAY,CAACC,MAAM,EAAEa,CAAC,CAAC;QACzB,CAAC,CAAC,OAAOU,CAAC,EAAE;AACVtD,UAAAA,OAAO,CAACC,GAAG,CAAC,OAAO,EAAEqD,CAAC,CAAC;AACzB,QAAA;AACF,MAAA,CAAC,CAAC,CACDC,EAAE,CAAC,OAAO,EAAExD,GAAG,IAAI;AAClBC,QAAAA,OAAO,CAACC,GAAG,CAAC,6BAA6B,EAAEF,GAAG,CAAC;AACjD,MAAA,CAAC,CACL,CAAC;AACH,IAAA,CAAC,GAAG;AAEV6C,IAAAA,CAAC,CAACW,EAAE,CAAC,KAAK,EAAE,MAAM;AAChB,MAAA,IAAItE,OAAO,EAAEe,OAAO,CAACC,GAAG,CAAC,qBAAqB,CAAC;MAC/CwB,OAAO,CAACqC,MAAM,EAAE;AAChBD,MAAAA,QAAQ,GAAG,IAAI;AACjB,IAAA,CAAC,CAAC;AAEFpC,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,OAAO,EAAE,MAAM;AACrC,MAAA,IAAIM,QAAQ,EAAE;MACdjB,CAAC,CAACqB,KAAK,EAAE;AACX,IAAA,CAAC,CAAC;AAEFxC,IAAAA,OAAO,CAACuC,YAAY,CAACT,EAAE,CAAC,QAAQ,EAAE,MAAM;AACtC,MAAA,IAAIM,QAAQ,EAAE;MACdjB,CAAC,CAACR,MAAM,EAAE;AACZ,IAAA,CAAC,CAAC;AACJ,EAAA;AAEA,EAAA,OAAO,MAAM;AACXuB,IAAAA,UAAU,EAAE;EACd,CAAC;AACH;;AC9FA;AACA;AACA;AACA,eAAeyH,0BAA0BA,CAACC,MAAM,EAAE;EAChD,IAAI;AACF;IACA,MAAMC,UAAU,GAAG,IAAIC,cAAc,CAACC,MAAM,CAACH,MAAM,CAAC;AACpD,IAAA,MAAMI,IAAI,GAAG,MAAMH,UAAU,CAACG,IAAI,EAAE;AACpC,IAAA,MAAMC,OAAO,GAAGD,IAAI,CAACC,OAAO,EAAEC,MAAM;AACpC,IAAA,MAAML,UAAU,CAACM,KAAK,EAAE;AAExB,IAAA,IAAIF,OAAO,EAAE;AACX,MAAA,MAAMG,YAAY,GAAGC,QAAQ,CAACJ,OAAO,CAAC1I,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACxD,MAAA,OAAO6I,YAAY;AACrB,IAAA;EACF,CAAC,CAAC,OAAOvI,CAAC,EAAE;AACV;IACA,IAAI;MACF,MAAMgI,UAAU,GAAG,IAAIS,cAAc,CAACP,MAAM,CAACH,MAAM,CAAC;AACpD,MAAA,MAAMI,IAAI,GAAG,MAAMH,UAAU,CAACG,IAAI,EAAE;AACpC,MAAA,MAAMC,OAAO,GAAGD,IAAI,CAACC,OAAO,EAAEC,MAAM;AACpC,MAAA,MAAML,UAAU,CAACM,KAAK,EAAE;AAExB,MAAA,IAAIF,OAAO,EAAE;AACX,QAAA,MAAMG,YAAY,GAAGC,QAAQ,CAACJ,OAAO,CAAC1I,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AACxD,QAAA,OAAO6I,YAAY;AACrB,MAAA;IACF,CAAC,CAAC,OAAOG,EAAE,EAAE;AACX;AAAA,IAAA;AAEJ,EAAA;;AAEA;AACA,EAAA,OAAO,CAAC;AACV;;AAEA;AACA;AACA;AACA;AACA;AACA;AACA,eAAeC,iBAAiBA,CAACC,cAAc,EAAEC,aAAa,EAAEC,YAAY,EAAE;AAC5E;EACA,IAAIF,cAAc,IAAI,OAAOA,cAAc,CAACT,IAAI,KAAK,UAAU,EAAE;AAC/D,IAAA,OAAOS,cAAc;AACvB,EAAA;AAEA,EAAA,MAAMb,MAAM,GAAGa,cAAc,IAAIC,aAAa;;AAE9C;EACA,IAAIC,YAAY,KAAK,CAAC,EAAE;AACtB,IAAA,OAAO,IAAIL,cAAc,CAACP,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA,CAAC,MAAM,IAAIe,YAAY,KAAK,CAAC,EAAE;AAC7B,IAAA,OAAO,IAAIb,cAAc,CAACC,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA;;AAEA;AACA,EAAA,MAAMQ,YAAY,GAAG,MAAMT,0BAA0B,CAACC,MAAM,CAAC;EAE7D,IAAIQ,YAAY,IAAI,CAAC,EAAE;AACrB,IAAA,OAAO,IAAIN,cAAc,CAACC,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA,CAAC,MAAM;AACL,IAAA,OAAO,IAAIU,cAAc,CAACP,MAAM,CAACH,MAAM,CAAC;AAC1C,EAAA;AACF;AAEe,eAAegB,WAAWA,CAAC;AACxCnN,EAAAA,WAAW,GAAG,KAAK;AACnBT,EAAAA,YAAY,EAAE6N,iBAAiB;AAC/B3N,EAAAA,YAAY,EAAE4N,iBAAiB;EAC/BC,kBAAkB;EAClBC,kBAAkB;EAClBC,mBAAmB;EACnBC,mBAAmB;AACnBlI,EAAAA,UAAU,GAAGnG,mBAAmB;AAChC8I,EAAAA,UAAU,GAAG7I,mBAAmB;EAChCsG,MAAM;EACNnD,QAAQ;AACRG,EAAAA,YAAY,GAAG,QAAQ;EACvBd,UAAU,GAAG,EAAE;AACfa,EAAAA,UAAU,GAAG,IAAI;EACjBlD,eAAe;EACfE,eAAe;EACfC,QAAQ;AACRE,EAAAA,gBAAgB,GAAG,KAAK;AACxBqL,EAAAA,aAAa,GAAG,KAAK;EACrBC,oBAAoB,GAAG,EAAE;EACzBrL,4BAA4B;EAC5BG,QAAQ;AACRkI,EAAAA,eAAe,GAAG,KAAK;EACvBF,KAAK;AACLnG,EAAAA,UAAU,GAAG,KAAK;EAClBW,SAAS;AACT1C,EAAAA,OAAO,GAAG;AACZ,CAAC,EAAE;AACD,EAAA,IAAI,OAAOL,eAAe,KAAK,WAAW,EAAE;IAC1C,MAAM2H,KAAK,CAAC,oCAAoC,CAAC;AACnD,EAAA;AAEA,EAAA,MAAMqG,mBAAmB,GAAG;AAC1BC,IAAAA,IAAI,EAAEC,OAAO,CAACC,GAAG,CAACC,iBAAiB,IAAI;GACxC;;AAED;AACA,EAAA,MAAMvO,YAAY,GAAG,MAAMwN,iBAAiB,CAC1CK,iBAAiB,IAAIE,kBAAkB,EACvCI,mBAAmB,EACnBF,mBACF,CAAC;AAED,EAAA,MAAM/N,YAAY,GAAG,MAAMsN,iBAAiB,CAC1CM,iBAAiB,IAAIE,kBAAkB,IAAIH,iBAAiB,IAAIE,kBAAkB,EAClFI,mBAAmB,EACnBD,mBACF,CAAC;AAED,EAAA,MAAMM,eAAe,GAAG,MAAM9C,uBAAuB,CAAC;IACpDxL,YAAY;IACZ+C,QAAQ;IACRG,YAAY;IACZd,UAAU;IACVC,UAAU;IACVnC,QAAQ;IACRuL,aAAa;IACbC,oBAAoB;AACpBpL,IAAAA;AACF,GAAC,CAAC;EAEF,MAAMiO,aAAa,GAAG1O,oBAAoB,CAAC;IACzCC,YAAY;IACZC,eAAe;IACfC,YAAY;IACZC,eAAe;IACfC,QAAQ,EAAEoO,eAAe,CAACpO,QAAQ;IAClCC,sBAAsB,EAAEmO,eAAe,CAAC/C,cAAc;IACtDnL,gBAAgB;IAChBC,4BAA4B;IAC5BC,OAAO;IACPC,WAAW;AACXC,IAAAA;AACF,GAAC,CAAC;EACF,MAAMsC,OAAO,GAAG8C,iBAAiB,CAAC;IAChC5F,YAAY;IACZC,eAAe;IACf6F,UAEF,CAAC,CAAC;EAEF,SAAS0I,oBAAoBA,GAAG;AAC9B,IAAA,IAAItL,YAAY,KAAK,QAAQ,IAAIA,YAAY,KAAK,KAAK,EAAE;AACvD,MAAA,MAAM0E,KAAK,CAAC,CAAA,0BAAA,EAA6B1E,YAAY,0BAA0B,CAAC;AAClF,IAAA;AACF,EAAA;EAEA,SAASuL,SAASA,GAAG;IACnB,IAAI,OAAO1L,QAAQ,KAAK,WAAW,IAAI,OAAOhD,eAAe,KAAK,WAAW,EAAE;MAC7E,MAAM6H,KAAK,CAAC,kEAAkE,CAAC;AACjF,IAAA;AAEA,IAAA,IACG,OAAO7E,QAAQ,KAAK,WAAW,IAAI,OAAOhD,eAAe,KAAK,WAAW,IACzE,OAAOgD,QAAQ,KAAK,WAAW,IAAI,OAAOmD,MAAM,KAAK,WAAY,IACjE,OAAOnG,eAAe,KAAK,WAAW,IAAI,OAAOmG,MAAM,KAAK,WAAY,EACzE;MACA,MAAM0B,KAAK,CAAC,oEAAoE,CAAC;AACnF,IAAA;AAEA,IAAA,IAAI,OAAO7E,QAAQ,KAAK,WAAW,EAAE;AACnCyL,MAAAA,oBAAoB,EAAE;AACtB,MAAA,OAAO3L,iBAAiB,CACtBC,OAAO,EACPC,QAAQ,EACRC,SAAS,EACTC,UAAU,EACV3C,OAAO,EACP+B,UAAU,EACVa,YAAY,EACZd,UACF,CAAC;AACH,IAAA;AAEA,IAAA,IAAI,OAAOrC,eAAe,KAAK,WAAW,EAAE;AAC1C,MAAA,OAAOwI,kBAAkB,CACvBzF,OAAO,EACP/C,eAAe,EACfiD,SAAS,EACTlD,YAAY,EACZ0I,KAAK,EACLC,UAAU,EACVC,eACF,CAAC;AACH,IAAA;AAEA,IAAA,IAAI,OAAOxC,MAAM,KAAK,WAAW,EAAE;AACjCsI,MAAAA,oBAAoB,EAAE;AACtB,MAAA,OAAOhC,mBAAmB,CACxB1J,OAAO,EACPoD,MAAM,EACNlD,SAAS,EACTC,UAAU,EACV3C,OAAO,EACP+B,UAAU,EACVa,YAAY,EACZd,UACF,CAAC;AACH,IAAA;AAEA,IAAA,OAAO,IAAI;AACb,EAAA;AAEA,EAAA,MAAMsM,MAAM,GAAGD,SAAS,EAAE;EAE1B,IAAI;IACF,MAAMjN,WAAW,GAAG,MAAMxB,YAAY,CAACa,OAAO,CAACY,MAAM,CAAC;AAAEV,MAAAA,KAAK,EAAEd;AAAgB,KAAC,CAAC;IAEjF,IAAIuB,WAAW,KAAK,KAAK,EAAE;MACzB,MAAM+M,aAAa,EAAE;AACrBG,MAAAA,MAAM,EAAE;AACV,IAAA,CAAC,MAAM,IAAInO,WAAW,KAAK,IAAI,EAAE;AAC/B,MAAA,MAAMP,YAAY,CAACa,OAAO,CAACa,MAAM,CAAC;AAAEX,QAAAA,KAAK,EAAEd;AAAgB,OAAC,CAAC;MAC7D,MAAMsO,aAAa,EAAE;AACrBG,MAAAA,MAAM,EAAE;AACV,IAAA,CAAC,MAAM;AACLA,MAAAA,MAAM,EAAE;AACV,IAAA;EACF,CAAC,CAAC,OAAOlJ,KAAK,EAAE;AACdnE,IAAAA,OAAO,CAACmE,KAAK,CAAC,iCAAiC,EAAEA,KAAK,CAAC;AACzD,EAAA,CAAC,SAAS;AACR;AAAA,EAAA;EAGF,OAAO;IAAEmJ,MAAM,EAAE7L,OAAO,CAACuC;GAAc;AACzC;;;;"}

package/index.d.ts CHANGED Viewed

@@ -25,6 +25,35 @@ export interface TransformContext {
   fileName?: string;
 }
+/**
+ * CSV parser options (subset of csv-parse options plus passthrough support)
+ */
+export interface CsvOptions {
+  delimiter?: string;
+  quote?: string;
+  escape?: string;
+  columns?: boolean | string[] | ((header: string[]) => string[]);
+  bom?: boolean;
+  trim?: boolean;
+  skip_empty_lines?: boolean;
+  from_line?: number;
+  [key: string]: any;
+}
+/**
+ * Mapping inference options for _text_structure/find_structure
+ */
+export interface InferMappingsOptions {
+  sampleBytes?: number;
+  lines_to_sample?: number;
+  timeout?: string;
+  charset?: string;
+  delimiter?: string;
+  quote?: string;
+  has_header_row?: boolean;
+  [key: string]: any;
+}
 /**
  * Transform function that processes each document
  * @param doc - The source document
@@ -112,8 +141,20 @@ export interface TransformerOptions {
    */
   stream?: Readable;
+  /**
+   * Source format for file/stream ingestion
+   * @default 'ndjson'
+   */
+  sourceFormat?: 'ndjson' | 'csv';
+  /**
+   * CSV parser options when sourceFormat is 'csv'
+   */
+  csvOptions?: CsvOptions;
   /**
    * Regular expression to split lines in file/stream
+   * Used only when sourceFormat is 'ndjson'
    * @default /\n/
    */
   splitRegex?: RegExp;
@@ -136,6 +177,20 @@ export interface TransformerOptions {
    */
   mappingsOverride?: boolean;
+  /**
+   * Infer mappings for file sources via _text_structure/find_structure
+   * Ignored when mappings is explicitly provided
+   * If inference returns an ingest pipeline and pipeline is not explicitly set,
+   * the inferred pipeline is created and applied as default index pipeline.
+   * @default false
+   */
+  inferMappings?: boolean;
+  /**
+   * Options for mapping inference via _text_structure/find_structure
+   */
+  inferMappingsOptions?: InferMappingsOptions;
   /**
    * Field limit for target index (index.mapping.total_fields.limit setting)
    */
@@ -159,7 +214,9 @@ export interface TransformerOptions {
   query?: QueryDslQueryContainer;
   /**
-   * Skip the first line of the source file (e.g., CSV header)
+   * Skip header line for file/stream ingestion
+   * - NDJSON: skips the first non-empty line
+   * - CSV: skips the first data line only when csvOptions.columns does not consume headers
    * @default false
    */
   skipHeader?: boolean;

package/package.json CHANGED Viewed

@@ -30,7 +30,7 @@
   "license": "Apache-2.0",
   "author": "Walter M. Rafelsberger <walter@rafelsberger.at>",
   "contributors": [],
-  "version": "1.0.2",
+  "version": "1.1.0",
   "engines": {
     "node": ">=22.0.0"
   },
@@ -39,6 +39,7 @@
   "types": "index.d.ts",
   "dependencies": {
     "cli-progress": "^3.12.0",
+    "csv-parse": "^5.6.0",
     "es8": "npm:@elastic/elasticsearch@^8.17.0",
     "es9": "npm:@elastic/elasticsearch@^9.2.0",
     "event-stream": "^3.3.4",