@sweetoburrito/backstage-plugin-ai-assistant-backend-module-ingestor-azure-devops 0.0.0-snapshot-20260109100733 → 0.0.0-snapshot-20260112102353
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -111,28 +111,40 @@ const createRepositoryIngestor = async ({
|
|
|
111
111
|
for (let index = 0; index < itemsBatch.length; index++) {
|
|
112
112
|
const item = itemsBatch[index];
|
|
113
113
|
const globalIndex = batchStart + index;
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
114
|
+
try {
|
|
115
|
+
const [content, lastUpdated] = await Promise.all([
|
|
116
|
+
azureDevOpsService.getRepoItemContent(repository.id, item.path),
|
|
117
|
+
azureDevOpsService.getRepoItemLastUpdated(
|
|
118
|
+
repository.id,
|
|
119
|
+
item.path
|
|
120
|
+
)
|
|
121
|
+
]);
|
|
122
|
+
const completionStats = backstagePluginAiAssistantCommon.getProgressStats(
|
|
123
|
+
globalIndex + 1,
|
|
124
|
+
items.length
|
|
125
|
+
);
|
|
126
|
+
logger.info(
|
|
127
|
+
`Retrieved content for Azure DevOps item: ${item.path} in repository: "${repository.name}" [Progress: ${completionStats.completed}/${completionStats.total} (${completionStats.percentage}%) completed of repository]`
|
|
128
|
+
);
|
|
129
|
+
const text = await backstagePluginAiAssistantNode.streamToString(content);
|
|
130
|
+
const document = {
|
|
131
|
+
metadata: {
|
|
132
|
+
source: module$1.MODULE_ID,
|
|
133
|
+
id: `${repository.id}:${item.path}`,
|
|
134
|
+
url: item.url,
|
|
135
|
+
organization: azureDevOpsService.organization,
|
|
136
|
+
project: azureDevOpsService.project,
|
|
137
|
+
repository: repository.name
|
|
138
|
+
},
|
|
139
|
+
content: text,
|
|
140
|
+
lastUpdated
|
|
141
|
+
};
|
|
142
|
+
documents.push(document);
|
|
143
|
+
} catch (error) {
|
|
144
|
+
logger.warn(
|
|
145
|
+
`Failed to retrieve content for Azure DevOps item: ${item.path} in repository: "${repository.name}". Error: ${error}. Skipping item and continuing.`
|
|
146
|
+
);
|
|
147
|
+
}
|
|
136
148
|
}
|
|
137
149
|
await saveDocumentsBatch(documents);
|
|
138
150
|
totalDocumentsIngested += documents.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repository.cjs.js","sources":["../../../src/services/ingestor/repository.ts"],"sourcesContent":["import {\n LoggerService,\n RootConfigService,\n} from '@backstage/backend-plugin-api';\nimport { DEFAULT_FILE_TYPES } from '../../constants/default-file-types';\nimport {\n EmbeddingDocument,\n IngestorOptions,\n streamToString,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { AzureDevOpsService } from '../azure-devops';\nimport { Config } from '../../../config';\nimport { MODULE_ID } from '../../constants/module';\nimport {\n getProgressStats,\n createPathFilter,\n validateExclusionPatterns,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-common';\nimport { DEFAULT_REPO_FILE_BATCH_SIZE } from '../../constants/default-repo-file-batch-size';\nimport { DEFAULT_PATH_EXCLUSIONS } from '../../constants/default-path-exclusions';\nimport {\n GitItem,\n GitRepository,\n} from 'azure-devops-node-api/interfaces/GitInterfaces';\n\ntype RepositoryIngestorOptions = {\n config: RootConfigService;\n logger: LoggerService;\n azureDevOpsService: AzureDevOpsService;\n};\n\nexport const createRepositoryIngestor = async ({\n config,\n logger,\n azureDevOpsService,\n}: RepositoryIngestorOptions) => {\n // Get configuration values\n const repositoriesFilter = config.getOptional<\n Config['aiAssistant']['ingestors']['azureDevOps']['repositories']\n >('aiAssistant.ingestors.azureDevOps.repositories');\n\n // Process and validate repository filters\n type RepositoryMatcher = {\n value: string;\n regex: RegExp;\n fileTypes?: string[];\n pathExclusions?: string[];\n };\n\n const includeMatchers: RepositoryMatcher[] = [];\n const excludeMatchers: RepositoryMatcher[] = [];\n\n if (repositoriesFilter?.include) {\n for (const filter of repositoriesFilter.include) {\n try {\n // All strings are treated as regular expression patterns; escape special characters for exact literal matches\n const regex = new RegExp(filter.name);\n includeMatchers.push({\n value: filter.name,\n regex,\n fileTypes: filter.fileTypes,\n pathExclusions: filter.pathExclusions,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in repository include '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid repository include pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (repositoriesFilter?.exclude) {\n for (const filter of repositoriesFilter.exclude) {\n try {\n // All strings are valid regex - plain strings match exactly, patterns match as regex\n const regex = new RegExp(filter.name);\n excludeMatchers.push({\n value: filter.name,\n regex,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in repository exclude '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid repository exclude pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (includeMatchers.length > 0) {\n logger.info(\n `Repository include filters: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n if (excludeMatchers.length > 0) {\n logger.info(\n `Repository exclude filters: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n\n // Default to common file types if none are specified\n const fileTypes =\n config.getOptionalStringArray(\n 'aiAssistant.ingestors.azureDevOps.fileTypes',\n ) ?? DEFAULT_FILE_TYPES;\n\n // Get batch size for processing repository items (default to 50 items per batch)\n const itemsBatchSize =\n config.getOptionalNumber(\n 'aiAssistant.ingestors.azureDevOps.filesBatchSize', // Reuse the same config for consistency\n ) ?? DEFAULT_REPO_FILE_BATCH_SIZE;\n\n // Get global path exclusion patterns from configuration, defaulting to predefined patterns\n const globalPathExclusions =\n config.getOptionalStringArray(\n 'aiAssistant.ingestors.azureDevOps.pathExclusions',\n ) ?? DEFAULT_PATH_EXCLUSIONS;\n\n // Validate exclusion patterns\n const validation = validateExclusionPatterns(globalPathExclusions);\n if (!validation.isValid) {\n logger.error(\n `Invalid path exclusion patterns in Azure DevOps ingestor configuration: ${validation.errors.join(\n ', ',\n )}`,\n );\n throw new Error(\n `Invalid path exclusion patterns: ${validation.errors.join(', ')}`,\n );\n }\n if (validation.warnings.length > 0) {\n logger.warn(\n `Path exclusion pattern warnings: ${validation.warnings.join(', ')}`,\n );\n }\n\n /**\n * Ingest Azure DevOps repository items in batches\n * @param repository - The repository to ingest items from\n * @param items - The list of items to ingest from the repository\n * @param saveDocumentsBatch - Function to save a batch of embedding documents\n * @returns Total number of documents ingested and sent for embedding from the repository\n */\n const ingestRepoByFileBatch = async ({\n repository,\n items,\n saveDocumentsBatch,\n }: {\n repository: GitRepository;\n items: GitItem[];\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'];\n }) => {\n logger.info(\n `Processing ${items.length} items from repository \"${repository.name}\" in batches of ${itemsBatchSize}`,\n );\n\n logger.debug(`Items: ${JSON.stringify(items, null, 2)}`);\n\n let totalDocumentsIngested = 0;\n\n // Process items in batches to manage memory and performance\n const totalBatches = Math.ceil(items.length / itemsBatchSize);\n\n for (\n let batchStart = 0;\n batchStart < items.length;\n batchStart += itemsBatchSize\n ) {\n const batchEnd = Math.min(batchStart + itemsBatchSize, items.length);\n const itemsBatch = items.slice(batchStart, batchEnd);\n const batchNumber = Math.floor(batchStart / itemsBatchSize) + 1;\n\n logger.info(\n `Processing batch ${batchNumber}/${totalBatches} (${itemsBatch.length} items) for repository \"${repository.name}\"`,\n );\n\n // Generate embedding documents for each item in the current batch\n const documents: EmbeddingDocument[] = [];\n\n for (let index = 0; index < itemsBatch.length; index++) {\n const item = itemsBatch[index];\n const globalIndex = batchStart + index;\n\n const [content, lastUpdated] = await Promise.all([\n azureDevOpsService.getRepoItemContent(repository.id!, item.path!),\n azureDevOpsService.getRepoItemLastUpdated(repository.id!, item.path!),\n ]);\n\n const completionStats = getProgressStats(globalIndex + 1, items.length);\n\n logger.info(\n `Retrieved content for Azure DevOps item: ${item.path} in repository: \"${repository.name}\" [Progress: ${completionStats.completed}/${completionStats.total} (${completionStats.percentage}%) completed of repository]`,\n );\n\n const text = await streamToString(content);\n\n const document: EmbeddingDocument = {\n metadata: {\n source: MODULE_ID,\n id: `${repository.id}:${item.path}`,\n url: item.url!,\n organization: azureDevOpsService.organization,\n project: azureDevOpsService.project,\n repository: repository.name!,\n },\n content: text,\n lastUpdated,\n };\n\n documents.push(document);\n }\n\n // Save the current batch of documents\n await saveDocumentsBatch(documents);\n\n totalDocumentsIngested += documents.length;\n\n logger.info(\n `Batch ${batchNumber}/${totalBatches} completed: ${documents.length} documents ingested for Azure DevOps repository: ${repository.name}`,\n );\n }\n\n return { totalDocumentsIngested };\n };\n\n /** Ingest Azure DevOps repositories in batches\n * @param saveDocumentsBatch - Function to save a batch of embedding documents\n * @returns void\n */\n const ingestRepositoriesBatch = async (\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n ) => {\n const repositoriesList = await azureDevOpsService.getRepos();\n\n if (repositoriesList.length === 0) {\n logger.warn('No repositories found in the Azure DevOps project');\n return;\n }\n\n // Filter repositories using matchers\n let repositoriesToIngest = repositoriesList;\n\n // If include matchers exist, only include repos that match at least one\n if (includeMatchers.length > 0) {\n logger.info(\n `Include filter found. Only including repositories matching the following patterns for ingestion: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n repositoriesToIngest = repositoriesToIngest.filter(repo => {\n return includeMatchers.some(matcher => matcher.regex!.test(repo.name!));\n });\n }\n\n // Apply exclusions\n if (excludeMatchers.length > 0) {\n logger.info(\n `Exclude filter found. Excluding repositories matching the following patterns from ingestion: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n const excludedRepos = repositoriesToIngest.filter(repo => {\n return excludeMatchers.some(matcher => matcher.regex!.test(repo.name!));\n });\n\n if (excludedRepos.length > 0) {\n logger.info(\n `Excluding repositories: ${excludedRepos\n .map(r => r.name)\n .join(', ')}`,\n );\n }\n repositoriesToIngest = repositoriesToIngest.filter(repo => {\n return !excludeMatchers.some(matcher =>\n matcher.regex!.test(repo.name!),\n );\n });\n }\n\n if (repositoriesToIngest.length === 0) {\n logger.warn(\n 'No repositories found for ingestion after applying the filter',\n );\n return;\n }\n\n logger.debug(\n `Repositories to ingest: ${repositoriesToIngest\n .map(r => r.name)\n .join(', ')}`,\n );\n\n logger.info(\n `Ingesting ${repositoriesToIngest.length} repositories from Azure DevOps`,\n );\n\n // Get items from each repository and create documents to be embedded\n for (const repo of repositoriesToIngest) {\n logger.info(\n `Beginning ingestion for repository: ${repo.name} (${repo.id})`,\n );\n\n // Find the matching include matcher for this repository\n const matchingMatcher = includeMatchers.find(matcher =>\n matcher.regex!.test(repo.name!),\n );\n\n // Determine the file types to use for this repository or use default\n const repositoryFileTypesFilter = matchingMatcher?.fileTypes ?? fileTypes;\n\n // Determine the path exclusions to use for this repository or use global default\n const repositoryPathExclusions =\n matchingMatcher?.pathExclusions ?? globalPathExclusions;\n\n logger.info(\n `Processing file types for repository ${\n repo.name\n }: [${repositoryFileTypesFilter.join(', ')}]`,\n );\n\n logger.info(\n `Using path exclusions for repository ${\n repo.name\n }: [${repositoryPathExclusions.join(', ')}]`,\n );\n\n // Get the items to be ingested from the repository based on the file types filter\n let items = await azureDevOpsService.getRepoItems(\n repo.id!,\n repositoryFileTypesFilter,\n );\n\n // Apply path exclusion filtering\n const pathFilter = createPathFilter({\n exclusionPatterns: repositoryPathExclusions,\n });\n\n const originalItemCount = items.length;\n\n // Log excluded items for debugging\n const excludedItems = items.filter(\n item => item.path && pathFilter.shouldExcludePath(item.path),\n );\n\n if (excludedItems.length > 0) {\n logger.debug(\n `Items excluded from repository ${repo.name}: ${excludedItems\n .map(i => i.path)\n .join(', ')}`,\n );\n }\n\n items = pathFilter.filterFiles(items);\n const filteredItemCount = originalItemCount - items.length;\n\n if (filteredItemCount > 0) {\n logger.info(\n `Filtered out ${filteredItemCount} items from repository ${repo.name} based on path exclusion patterns`,\n );\n }\n\n if (items.length === 0) {\n logger.warn(\n `No items found for ingestion in the Azure DevOps repository ${\n repo.name\n } (${\n repo.id\n }) with the specified file types filter: [${repositoryFileTypesFilter.join(\n ', ',\n )}]`,\n );\n continue;\n }\n\n const { totalDocumentsIngested } = await ingestRepoByFileBatch({\n repository: repo,\n items,\n saveDocumentsBatch,\n });\n\n if (totalDocumentsIngested === 0) {\n logger.warn(\n `No documents were ingested and sent for embedding from the Azure DevOps repository ${repo.name} (${repo.id})`,\n );\n continue;\n }\n\n logger.info(\n `Repository ingestion completed: ${totalDocumentsIngested} total documents ingested and sent for embedding for Azure DevOps repository: ${repo.name}`,\n );\n }\n };\n\n return { ingestRepositoriesBatch };\n};\n"],"names":["DEFAULT_FILE_TYPES","DEFAULT_REPO_FILE_BATCH_SIZE","DEFAULT_PATH_EXCLUSIONS","validateExclusionPatterns","getProgressStats","streamToString","MODULE_ID","createPathFilter"],"mappings":";;;;;;;;;AA+BO,MAAM,2BAA2B,OAAO;AAAA,EAC7C,MAAA;AAAA,EACA,MAAA;AAAA,EACA;AACF,CAAA,KAAiC;AAE/B,EAAA,MAAM,kBAAA,GAAqB,MAAA,CAAO,WAAA,CAEhC,gDAAgD,CAAA;AAUlD,EAAA,MAAM,kBAAuC,EAAC;AAC9C,EAAA,MAAM,kBAAuC,EAAC;AAE9C,EAAA,IAAI,oBAAoB,OAAA,EAAS;AAC/B,IAAA,KAAA,MAAW,MAAA,IAAU,mBAAmB,OAAA,EAAS;AAC/C,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd,KAAA;AAAA,UACA,WAAW,MAAA,CAAO,SAAA;AAAA,UAClB,gBAAgB,MAAA,CAAO;AAAA,SACxB,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,kDAAA,EAAqD,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC7E;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,oCAAA,EAAuC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC/D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,oBAAoB,OAAA,EAAS;AAC/B,IAAA,KAAA,MAAW,MAAA,IAAU,mBAAmB,OAAA,EAAS;AAC/C,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd;AAAA,SACD,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,kDAAA,EAAqD,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC7E;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,oCAAA,EAAuC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC/D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,4BAAA,EAA+B,eAAA,CAC5B,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AACA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,4BAAA,EAA+B,eAAA,CAC5B,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AAGA,EAAA,MAAM,YACJ,MAAA,CAAO,sBAAA;AAAA,IACL;AAAA,GACF,IAAKA,mCAAA;AAGP,EAAA,MAAM,iBACJ,MAAA,CAAO,iBAAA;AAAA,IACL;AAAA;AAAA,GACF,IAAKC,qDAAA;AAGP,EAAA,MAAM,uBACJ,MAAA,CAAO,sBAAA;AAAA,IACL;AAAA,GACF,IAAKC,6CAAA;AAGP,EAAA,MAAM,UAAA,GAAaC,2DAA0B,oBAAoB,CAAA;AACjE,EAAA,IAAI,CAAC,WAAW,OAAA,EAAS;AACvB,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,wEAAA,EAA2E,WAAW,MAAA,CAAO,IAAA;AAAA,QAC3F;AAAA,OACD,CAAA;AAAA,KACH;AACA,IAAA,MAAM,IAAI,KAAA;AAAA,MACR,CAAA,iCAAA,EAAoC,UAAA,CAAW,MAAA,CAAO,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KAClE;AAAA,EACF;AACA,EAAA,IAAI,UAAA,CAAW,QAAA,CAAS,MAAA,GAAS,CAAA,EAAG;AAClC,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,iCAAA,EAAoC,UAAA,CAAW,QAAA,CAAS,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACpE;AAAA,EACF;AASA,EAAA,MAAM,wBAAwB,OAAO;AAAA,IACnC,UAAA;AAAA,IACA,KAAA;AAAA,IACA;AAAA,GACF,KAIM;AACJ,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,cAAc,KAAA,CAAM,MAAM,2BAA2B,UAAA,CAAW,IAAI,mBAAmB,cAAc,CAAA;AAAA,KACvG;AAEA,IAAA,MAAA,CAAO,KAAA,CAAM,UAAU,IAAA,CAAK,SAAA,CAAU,OAAO,IAAA,EAAM,CAAC,CAAC,CAAA,CAAE,CAAA;AAEvD,IAAA,IAAI,sBAAA,GAAyB,CAAA;AAG7B,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,SAAS,cAAc,CAAA;AAE5D,IAAA,KAAA,IACM,aAAa,CAAA,EACjB,UAAA,GAAa,KAAA,CAAM,MAAA,EACnB,cAAc,cAAA,EACd;AACA,MAAA,MAAM,WAAW,IAAA,CAAK,GAAA,CAAI,UAAA,GAAa,cAAA,EAAgB,MAAM,MAAM,CAAA;AACnE,MAAA,MAAM,UAAA,GAAa,KAAA,CAAM,KAAA,CAAM,UAAA,EAAY,QAAQ,CAAA;AACnD,MAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,UAAA,GAAa,cAAc,CAAA,GAAI,CAAA;AAE9D,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,iBAAA,EAAoB,WAAW,CAAA,CAAA,EAAI,YAAY,KAAK,UAAA,CAAW,MAAM,CAAA,wBAAA,EAA2B,UAAA,CAAW,IAAI,CAAA,CAAA;AAAA,OACjH;AAGA,MAAA,MAAM,YAAiC,EAAC;AAExC,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,UAAA,CAAW,QAAQ,KAAA,EAAA,EAAS;AACtD,QAAA,MAAM,IAAA,GAAO,WAAW,KAAK,CAAA;AAC7B,QAAA,MAAM,cAAc,UAAA,GAAa,KAAA;AAEjC,QAAA,MAAM,CAAC,OAAA,EAAS,WAAW,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,UAC/C,kBAAA,CAAmB,kBAAA,CAAmB,UAAA,CAAW,EAAA,EAAK,KAAK,IAAK,CAAA;AAAA,UAChE,kBAAA,CAAmB,sBAAA,CAAuB,UAAA,CAAW,EAAA,EAAK,KAAK,IAAK;AAAA,SACrE,CAAA;AAED,QAAA,MAAM,eAAA,GAAkBC,iDAAA,CAAiB,WAAA,GAAc,CAAA,EAAG,MAAM,MAAM,CAAA;AAEtE,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,yCAAA,EAA4C,IAAA,CAAK,IAAI,CAAA,iBAAA,EAAoB,WAAW,IAAI,CAAA,aAAA,EAAgB,eAAA,CAAgB,SAAS,CAAA,CAAA,EAAI,eAAA,CAAgB,KAAK,CAAA,EAAA,EAAK,gBAAgB,UAAU,CAAA,2BAAA;AAAA,SAC3L;AAEA,QAAA,MAAM,IAAA,GAAO,MAAMC,6CAAA,CAAe,OAAO,CAAA;AAEzC,QAAA,MAAM,QAAA,GAA8B;AAAA,UAClC,QAAA,EAAU;AAAA,YACR,MAAA,EAAQC,kBAAA;AAAA,YACR,IAAI,CAAA,EAAG,UAAA,CAAW,EAAE,CAAA,CAAA,EAAI,KAAK,IAAI,CAAA,CAAA;AAAA,YACjC,KAAK,IAAA,CAAK,GAAA;AAAA,YACV,cAAc,kBAAA,CAAmB,YAAA;AAAA,YACjC,SAAS,kBAAA,CAAmB,OAAA;AAAA,YAC5B,YAAY,UAAA,CAAW;AAAA,WACzB;AAAA,UACA,OAAA,EAAS,IAAA;AAAA,UACT;AAAA,SACF;AAEA,QAAA,SAAA,CAAU,KAAK,QAAQ,CAAA;AAAA,MACzB;AAGA,MAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,MAAA,sBAAA,IAA0B,SAAA,CAAU,MAAA;AAEpC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,MAAA,EAAS,WAAW,CAAA,CAAA,EAAI,YAAY,eAAe,SAAA,CAAU,MAAM,CAAA,iDAAA,EAAoD,UAAA,CAAW,IAAI,CAAA;AAAA,OACxI;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,sBAAA,EAAuB;AAAA,EAClC,CAAA;AAMA,EAAA,MAAM,uBAAA,GAA0B,OAC9B,kBAAA,KACG;AACH,IAAA,MAAM,gBAAA,GAAmB,MAAM,kBAAA,CAAmB,QAAA,EAAS;AAE3D,IAAA,IAAI,gBAAA,CAAiB,WAAW,CAAA,EAAG;AACjC,MAAA,MAAA,CAAO,KAAK,mDAAmD,CAAA;AAC/D,MAAA;AAAA,IACF;AAGA,IAAA,IAAI,oBAAA,GAAuB,gBAAA;AAG3B,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,iGAAA,EAAoG,eAAA,CACjG,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,oBAAA,GAAuB,oBAAA,CAAqB,OAAO,CAAA,IAAA,KAAQ;AACzD,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAAA,IACH;AAGA,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,6FAAA,EAAgG,eAAA,CAC7F,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,MAAM,aAAA,GAAgB,oBAAA,CAAqB,MAAA,CAAO,CAAA,IAAA,KAAQ;AACxD,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAED,MAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,wBAAA,EAA2B,cACxB,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CACf,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AACA,MAAA,oBAAA,GAAuB,oBAAA,CAAqB,OAAO,CAAA,IAAA,KAAQ;AACzD,QAAA,OAAO,CAAC,eAAA,CAAgB,IAAA;AAAA,UAAK,CAAA,OAAA,KAC3B,OAAA,CAAQ,KAAA,CAAO,IAAA,CAAK,KAAK,IAAK;AAAA,SAChC;AAAA,MACF,CAAC,CAAA;AAAA,IACH;AAEA,IAAA,IAAI,oBAAA,CAAqB,WAAW,CAAA,EAAG;AACrC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL;AAAA,OACF;AACA,MAAA;AAAA,IACF;AAEA,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,wBAAA,EAA2B,qBACxB,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CACf,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAEA,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,UAAA,EAAa,qBAAqB,MAAM,CAAA,+BAAA;AAAA,KAC1C;AAGA,IAAA,KAAA,MAAW,QAAQ,oBAAA,EAAsB;AACvC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oCAAA,EAAuC,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,OAC9D;AAGA,MAAA,MAAM,kBAAkB,eAAA,CAAgB,IAAA;AAAA,QAAK,CAAA,OAAA,KAC3C,OAAA,CAAQ,KAAA,CAAO,IAAA,CAAK,KAAK,IAAK;AAAA,OAChC;AAGA,MAAA,MAAM,yBAAA,GAA4B,iBAAiB,SAAA,IAAa,SAAA;AAGhE,MAAA,MAAM,wBAAA,GACJ,iBAAiB,cAAA,IAAkB,oBAAA;AAErC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,wCACE,IAAA,CAAK,IACP,MAAM,yBAAA,CAA0B,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,OAC5C;AAEA,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,wCACE,IAAA,CAAK,IACP,MAAM,wBAAA,CAAyB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,OAC3C;AAGA,MAAA,IAAI,KAAA,GAAQ,MAAM,kBAAA,CAAmB,YAAA;AAAA,QACnC,IAAA,CAAK,EAAA;AAAA,QACL;AAAA,OACF;AAGA,MAAA,MAAM,aAAaC,iDAAA,CAAiB;AAAA,QAClC,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAED,MAAA,MAAM,oBAAoB,KAAA,CAAM,MAAA;AAGhC,MAAA,MAAM,gBAAgB,KAAA,CAAM,MAAA;AAAA,QAC1B,UAAQ,IAAA,CAAK,IAAA,IAAQ,UAAA,CAAW,iBAAA,CAAkB,KAAK,IAAI;AAAA,OAC7D;AAEA,MAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,+BAAA,EAAkC,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,aAAA,CAC7C,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,IAAI,CAAA,CACf,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AAEA,MAAA,KAAA,GAAQ,UAAA,CAAW,YAAY,KAAK,CAAA;AACpC,MAAA,MAAM,iBAAA,GAAoB,oBAAoB,KAAA,CAAM,MAAA;AAEpD,MAAA,IAAI,oBAAoB,CAAA,EAAG;AACzB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,aAAA,EAAgB,iBAAiB,CAAA,uBAAA,EAA0B,IAAA,CAAK,IAAI,CAAA,iCAAA;AAAA,SACtE;AAAA,MACF;AAEA,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,+DACE,IAAA,CAAK,IACP,KACE,IAAA,CAAK,EACP,4CAA4C,yBAAA,CAA0B,IAAA;AAAA,YACpE;AAAA,WACD,CAAA,CAAA;AAAA,SACH;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,EAAE,sBAAA,EAAuB,GAAI,MAAM,qBAAA,CAAsB;AAAA,QAC7D,UAAA,EAAY,IAAA;AAAA,QACZ,KAAA;AAAA,QACA;AAAA,OACD,CAAA;AAED,MAAA,IAAI,2BAA2B,CAAA,EAAG;AAChC,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,mFAAA,EAAsF,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,SAC7G;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,gCAAA,EAAmC,sBAAsB,CAAA,8EAAA,EAAiF,IAAA,CAAK,IAAI,CAAA;AAAA,OACrJ;AAAA,IACF;AAAA,EACF,CAAA;AAEA,EAAA,OAAO,EAAE,uBAAA,EAAwB;AACnC;;;;"}
|
|
1
|
+
{"version":3,"file":"repository.cjs.js","sources":["../../../src/services/ingestor/repository.ts"],"sourcesContent":["import {\n LoggerService,\n RootConfigService,\n} from '@backstage/backend-plugin-api';\nimport { DEFAULT_FILE_TYPES } from '../../constants/default-file-types';\nimport {\n EmbeddingDocument,\n IngestorOptions,\n streamToString,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { AzureDevOpsService } from '../azure-devops';\nimport { Config } from '../../../config';\nimport { MODULE_ID } from '../../constants/module';\nimport {\n getProgressStats,\n createPathFilter,\n validateExclusionPatterns,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-common';\nimport { DEFAULT_REPO_FILE_BATCH_SIZE } from '../../constants/default-repo-file-batch-size';\nimport { DEFAULT_PATH_EXCLUSIONS } from '../../constants/default-path-exclusions';\nimport {\n GitItem,\n GitRepository,\n} from 'azure-devops-node-api/interfaces/GitInterfaces';\n\ntype RepositoryIngestorOptions = {\n config: RootConfigService;\n logger: LoggerService;\n azureDevOpsService: AzureDevOpsService;\n};\n\nexport const createRepositoryIngestor = async ({\n config,\n logger,\n azureDevOpsService,\n}: RepositoryIngestorOptions) => {\n // Get configuration values\n const repositoriesFilter = config.getOptional<\n Config['aiAssistant']['ingestors']['azureDevOps']['repositories']\n >('aiAssistant.ingestors.azureDevOps.repositories');\n\n // Process and validate repository filters\n type RepositoryMatcher = {\n value: string;\n regex: RegExp;\n fileTypes?: string[];\n pathExclusions?: string[];\n };\n\n const includeMatchers: RepositoryMatcher[] = [];\n const excludeMatchers: RepositoryMatcher[] = [];\n\n if (repositoriesFilter?.include) {\n for (const filter of repositoriesFilter.include) {\n try {\n // All strings are treated as regular expression patterns; escape special characters for exact literal matches\n const regex = new RegExp(filter.name);\n includeMatchers.push({\n value: filter.name,\n regex,\n fileTypes: filter.fileTypes,\n pathExclusions: filter.pathExclusions,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in repository include '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid repository include pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (repositoriesFilter?.exclude) {\n for (const filter of repositoriesFilter.exclude) {\n try {\n // All strings are valid regex - plain strings match exactly, patterns match as regex\n const regex = new RegExp(filter.name);\n excludeMatchers.push({\n value: filter.name,\n regex,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in repository exclude '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid repository exclude pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (includeMatchers.length > 0) {\n logger.info(\n `Repository include filters: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n if (excludeMatchers.length > 0) {\n logger.info(\n `Repository exclude filters: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n\n // Default to common file types if none are specified\n const fileTypes =\n config.getOptionalStringArray(\n 'aiAssistant.ingestors.azureDevOps.fileTypes',\n ) ?? DEFAULT_FILE_TYPES;\n\n // Get batch size for processing repository items (default to 50 items per batch)\n const itemsBatchSize =\n config.getOptionalNumber(\n 'aiAssistant.ingestors.azureDevOps.filesBatchSize', // Reuse the same config for consistency\n ) ?? DEFAULT_REPO_FILE_BATCH_SIZE;\n\n // Get global path exclusion patterns from configuration, defaulting to predefined patterns\n const globalPathExclusions =\n config.getOptionalStringArray(\n 'aiAssistant.ingestors.azureDevOps.pathExclusions',\n ) ?? DEFAULT_PATH_EXCLUSIONS;\n\n // Validate exclusion patterns\n const validation = validateExclusionPatterns(globalPathExclusions);\n if (!validation.isValid) {\n logger.error(\n `Invalid path exclusion patterns in Azure DevOps ingestor configuration: ${validation.errors.join(\n ', ',\n )}`,\n );\n throw new Error(\n `Invalid path exclusion patterns: ${validation.errors.join(', ')}`,\n );\n }\n if (validation.warnings.length > 0) {\n logger.warn(\n `Path exclusion pattern warnings: ${validation.warnings.join(', ')}`,\n );\n }\n\n /**\n * Ingest Azure DevOps repository items in batches\n * @param repository - The repository to ingest items from\n * @param items - The list of items to ingest from the repository\n * @param saveDocumentsBatch - Function to save a batch of embedding documents\n * @returns Total number of documents ingested and sent for embedding from the repository\n */\n const ingestRepoByFileBatch = async ({\n repository,\n items,\n saveDocumentsBatch,\n }: {\n repository: GitRepository;\n items: GitItem[];\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'];\n }) => {\n logger.info(\n `Processing ${items.length} items from repository \"${repository.name}\" in batches of ${itemsBatchSize}`,\n );\n\n logger.debug(`Items: ${JSON.stringify(items, null, 2)}`);\n\n let totalDocumentsIngested = 0;\n\n // Process items in batches to manage memory and performance\n const totalBatches = Math.ceil(items.length / itemsBatchSize);\n\n for (\n let batchStart = 0;\n batchStart < items.length;\n batchStart += itemsBatchSize\n ) {\n const batchEnd = Math.min(batchStart + itemsBatchSize, items.length);\n const itemsBatch = items.slice(batchStart, batchEnd);\n const batchNumber = Math.floor(batchStart / itemsBatchSize) + 1;\n\n logger.info(\n `Processing batch ${batchNumber}/${totalBatches} (${itemsBatch.length} items) for repository \"${repository.name}\"`,\n );\n\n // Generate embedding documents for each item in the current batch\n const documents: EmbeddingDocument[] = [];\n\n for (let index = 0; index < itemsBatch.length; index++) {\n const item = itemsBatch[index];\n const globalIndex = batchStart + index;\n\n try {\n const [content, lastUpdated] = await Promise.all([\n azureDevOpsService.getRepoItemContent(repository.id!, item.path!),\n azureDevOpsService.getRepoItemLastUpdated(\n repository.id!,\n item.path!,\n ),\n ]);\n\n const completionStats = getProgressStats(\n globalIndex + 1,\n items.length,\n );\n\n logger.info(\n `Retrieved content for Azure DevOps item: ${item.path} in repository: \"${repository.name}\" [Progress: ${completionStats.completed}/${completionStats.total} (${completionStats.percentage}%) completed of repository]`,\n );\n\n const text = await streamToString(content);\n\n const document: EmbeddingDocument = {\n metadata: {\n source: MODULE_ID,\n id: `${repository.id}:${item.path}`,\n url: item.url!,\n organization: azureDevOpsService.organization,\n project: azureDevOpsService.project,\n repository: repository.name!,\n },\n content: text,\n lastUpdated,\n };\n\n documents.push(document);\n } catch (error) {\n logger.warn(\n `Failed to retrieve content for Azure DevOps item: ${item.path} in repository: \"${repository.name}\". Error: ${error}. Skipping item and continuing.`,\n );\n // Continue to next item instead of crashing entire ingestion\n }\n }\n\n // Save the current batch of documents\n await saveDocumentsBatch(documents);\n\n totalDocumentsIngested += documents.length;\n\n logger.info(\n `Batch ${batchNumber}/${totalBatches} completed: ${documents.length} documents ingested for Azure DevOps repository: ${repository.name}`,\n );\n }\n\n return { totalDocumentsIngested };\n };\n\n /** Ingest Azure DevOps repositories in batches\n * @param saveDocumentsBatch - Function to save a batch of embedding documents\n * @returns void\n */\n const ingestRepositoriesBatch = async (\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n ) => {\n const repositoriesList = await azureDevOpsService.getRepos();\n\n if (repositoriesList.length === 0) {\n logger.warn('No repositories found in the Azure DevOps project');\n return;\n }\n\n // Filter repositories using matchers\n let repositoriesToIngest = repositoriesList;\n\n // If include matchers exist, only include repos that match at least one\n if (includeMatchers.length > 0) {\n logger.info(\n `Include filter found. Only including repositories matching the following patterns for ingestion: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n repositoriesToIngest = repositoriesToIngest.filter(repo => {\n return includeMatchers.some(matcher => matcher.regex!.test(repo.name!));\n });\n }\n\n // Apply exclusions\n if (excludeMatchers.length > 0) {\n logger.info(\n `Exclude filter found. Excluding repositories matching the following patterns from ingestion: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n const excludedRepos = repositoriesToIngest.filter(repo => {\n return excludeMatchers.some(matcher => matcher.regex!.test(repo.name!));\n });\n\n if (excludedRepos.length > 0) {\n logger.info(\n `Excluding repositories: ${excludedRepos\n .map(r => r.name)\n .join(', ')}`,\n );\n }\n repositoriesToIngest = repositoriesToIngest.filter(repo => {\n return !excludeMatchers.some(matcher =>\n matcher.regex!.test(repo.name!),\n );\n });\n }\n\n if (repositoriesToIngest.length === 0) {\n logger.warn(\n 'No repositories found for ingestion after applying the filter',\n );\n return;\n }\n\n logger.debug(\n `Repositories to ingest: ${repositoriesToIngest\n .map(r => r.name)\n .join(', ')}`,\n );\n\n logger.info(\n `Ingesting ${repositoriesToIngest.length} repositories from Azure DevOps`,\n );\n\n // Get items from each repository and create documents to be embedded\n for (const repo of repositoriesToIngest) {\n logger.info(\n `Beginning ingestion for repository: ${repo.name} (${repo.id})`,\n );\n\n // Find the matching include matcher for this repository\n const matchingMatcher = includeMatchers.find(matcher =>\n matcher.regex!.test(repo.name!),\n );\n\n // Determine the file types to use for this repository or use default\n const repositoryFileTypesFilter = matchingMatcher?.fileTypes ?? fileTypes;\n\n // Determine the path exclusions to use for this repository or use global default\n const repositoryPathExclusions =\n matchingMatcher?.pathExclusions ?? globalPathExclusions;\n\n logger.info(\n `Processing file types for repository ${\n repo.name\n }: [${repositoryFileTypesFilter.join(', ')}]`,\n );\n\n logger.info(\n `Using path exclusions for repository ${\n repo.name\n }: [${repositoryPathExclusions.join(', ')}]`,\n );\n\n // Get the items to be ingested from the repository based on the file types filter\n let items = await azureDevOpsService.getRepoItems(\n repo.id!,\n repositoryFileTypesFilter,\n );\n\n // Apply path exclusion filtering\n const pathFilter = createPathFilter({\n exclusionPatterns: repositoryPathExclusions,\n });\n\n const originalItemCount = items.length;\n\n // Log excluded items for debugging\n const excludedItems = items.filter(\n item => item.path && pathFilter.shouldExcludePath(item.path),\n );\n\n if (excludedItems.length > 0) {\n logger.debug(\n `Items excluded from repository ${repo.name}: ${excludedItems\n .map(i => i.path)\n .join(', ')}`,\n );\n }\n\n items = pathFilter.filterFiles(items);\n const filteredItemCount = originalItemCount - items.length;\n\n if (filteredItemCount > 0) {\n logger.info(\n `Filtered out ${filteredItemCount} items from repository ${repo.name} based on path exclusion patterns`,\n );\n }\n\n if (items.length === 0) {\n logger.warn(\n `No items found for ingestion in the Azure DevOps repository ${\n repo.name\n } (${\n repo.id\n }) with the specified file types filter: [${repositoryFileTypesFilter.join(\n ', ',\n )}]`,\n );\n continue;\n }\n\n const { totalDocumentsIngested } = await ingestRepoByFileBatch({\n repository: repo,\n items,\n saveDocumentsBatch,\n });\n\n if (totalDocumentsIngested === 0) {\n logger.warn(\n `No documents were ingested and sent for embedding from the Azure DevOps repository ${repo.name} (${repo.id})`,\n );\n continue;\n }\n\n logger.info(\n `Repository ingestion completed: ${totalDocumentsIngested} total documents ingested and sent for embedding for Azure DevOps repository: ${repo.name}`,\n );\n }\n };\n\n return { ingestRepositoriesBatch };\n};\n"],"names":["DEFAULT_FILE_TYPES","DEFAULT_REPO_FILE_BATCH_SIZE","DEFAULT_PATH_EXCLUSIONS","validateExclusionPatterns","getProgressStats","streamToString","MODULE_ID","createPathFilter"],"mappings":";;;;;;;;;AA+BO,MAAM,2BAA2B,OAAO;AAAA,EAC7C,MAAA;AAAA,EACA,MAAA;AAAA,EACA;AACF,CAAA,KAAiC;AAE/B,EAAA,MAAM,kBAAA,GAAqB,MAAA,CAAO,WAAA,CAEhC,gDAAgD,CAAA;AAUlD,EAAA,MAAM,kBAAuC,EAAC;AAC9C,EAAA,MAAM,kBAAuC,EAAC;AAE9C,EAAA,IAAI,oBAAoB,OAAA,EAAS;AAC/B,IAAA,KAAA,MAAW,MAAA,IAAU,mBAAmB,OAAA,EAAS;AAC/C,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd,KAAA;AAAA,UACA,WAAW,MAAA,CAAO,SAAA;AAAA,UAClB,gBAAgB,MAAA,CAAO;AAAA,SACxB,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,kDAAA,EAAqD,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC7E;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,oCAAA,EAAuC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC/D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,oBAAoB,OAAA,EAAS;AAC/B,IAAA,KAAA,MAAW,MAAA,IAAU,mBAAmB,OAAA,EAAS;AAC/C,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd;AAAA,SACD,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,kDAAA,EAAqD,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC7E;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,oCAAA,EAAuC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SAC/D;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,4BAAA,EAA+B,eAAA,CAC5B,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AACA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,4BAAA,EAA+B,eAAA,CAC5B,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AAGA,EAAA,MAAM,YACJ,MAAA,CAAO,sBAAA;AAAA,IACL;AAAA,GACF,IAAKA,mCAAA;AAGP,EAAA,MAAM,iBACJ,MAAA,CAAO,iBAAA;AAAA,IACL;AAAA;AAAA,GACF,IAAKC,qDAAA;AAGP,EAAA,MAAM,uBACJ,MAAA,CAAO,sBAAA;AAAA,IACL;AAAA,GACF,IAAKC,6CAAA;AAGP,EAAA,MAAM,UAAA,GAAaC,2DAA0B,oBAAoB,CAAA;AACjE,EAAA,IAAI,CAAC,WAAW,OAAA,EAAS;AACvB,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,wEAAA,EAA2E,WAAW,MAAA,CAAO,IAAA;AAAA,QAC3F;AAAA,OACD,CAAA;AAAA,KACH;AACA,IAAA,MAAM,IAAI,KAAA;AAAA,MACR,CAAA,iCAAA,EAAoC,UAAA,CAAW,MAAA,CAAO,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KAClE;AAAA,EACF;AACA,EAAA,IAAI,UAAA,CAAW,QAAA,CAAS,MAAA,GAAS,CAAA,EAAG;AAClC,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,iCAAA,EAAoC,UAAA,CAAW,QAAA,CAAS,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACpE;AAAA,EACF;AASA,EAAA,MAAM,wBAAwB,OAAO;AAAA,IACnC,UAAA;AAAA,IACA,KAAA;AAAA,IACA;AAAA,GACF,KAIM;AACJ,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,cAAc,KAAA,CAAM,MAAM,2BAA2B,UAAA,CAAW,IAAI,mBAAmB,cAAc,CAAA;AAAA,KACvG;AAEA,IAAA,MAAA,CAAO,KAAA,CAAM,UAAU,IAAA,CAAK,SAAA,CAAU,OAAO,IAAA,EAAM,CAAC,CAAC,CAAA,CAAE,CAAA;AAEvD,IAAA,IAAI,sBAAA,GAAyB,CAAA;AAG7B,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,SAAS,cAAc,CAAA;AAE5D,IAAA,KAAA,IACM,aAAa,CAAA,EACjB,UAAA,GAAa,KAAA,CAAM,MAAA,EACnB,cAAc,cAAA,EACd;AACA,MAAA,MAAM,WAAW,IAAA,CAAK,GAAA,CAAI,UAAA,GAAa,cAAA,EAAgB,MAAM,MAAM,CAAA;AACnE,MAAA,MAAM,UAAA,GAAa,KAAA,CAAM,KAAA,CAAM,UAAA,EAAY,QAAQ,CAAA;AACnD,MAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,UAAA,GAAa,cAAc,CAAA,GAAI,CAAA;AAE9D,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,iBAAA,EAAoB,WAAW,CAAA,CAAA,EAAI,YAAY,KAAK,UAAA,CAAW,MAAM,CAAA,wBAAA,EAA2B,UAAA,CAAW,IAAI,CAAA,CAAA;AAAA,OACjH;AAGA,MAAA,MAAM,YAAiC,EAAC;AAExC,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,UAAA,CAAW,QAAQ,KAAA,EAAA,EAAS;AACtD,QAAA,MAAM,IAAA,GAAO,WAAW,KAAK,CAAA;AAC7B,QAAA,MAAM,cAAc,UAAA,GAAa,KAAA;AAEjC,QAAA,IAAI;AACF,UAAA,MAAM,CAAC,OAAA,EAAS,WAAW,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,YAC/C,kBAAA,CAAmB,kBAAA,CAAmB,UAAA,CAAW,EAAA,EAAK,KAAK,IAAK,CAAA;AAAA,YAChE,kBAAA,CAAmB,sBAAA;AAAA,cACjB,UAAA,CAAW,EAAA;AAAA,cACX,IAAA,CAAK;AAAA;AACP,WACD,CAAA;AAED,UAAA,MAAM,eAAA,GAAkBC,iDAAA;AAAA,YACtB,WAAA,GAAc,CAAA;AAAA,YACd,KAAA,CAAM;AAAA,WACR;AAEA,UAAA,MAAA,CAAO,IAAA;AAAA,YACL,CAAA,yCAAA,EAA4C,IAAA,CAAK,IAAI,CAAA,iBAAA,EAAoB,WAAW,IAAI,CAAA,aAAA,EAAgB,eAAA,CAAgB,SAAS,CAAA,CAAA,EAAI,eAAA,CAAgB,KAAK,CAAA,EAAA,EAAK,gBAAgB,UAAU,CAAA,2BAAA;AAAA,WAC3L;AAEA,UAAA,MAAM,IAAA,GAAO,MAAMC,6CAAA,CAAe,OAAO,CAAA;AAEzC,UAAA,MAAM,QAAA,GAA8B;AAAA,YAClC,QAAA,EAAU;AAAA,cACR,MAAA,EAAQC,kBAAA;AAAA,cACR,IAAI,CAAA,EAAG,UAAA,CAAW,EAAE,CAAA,CAAA,EAAI,KAAK,IAAI,CAAA,CAAA;AAAA,cACjC,KAAK,IAAA,CAAK,GAAA;AAAA,cACV,cAAc,kBAAA,CAAmB,YAAA;AAAA,cACjC,SAAS,kBAAA,CAAmB,OAAA;AAAA,cAC5B,YAAY,UAAA,CAAW;AAAA,aACzB;AAAA,YACA,OAAA,EAAS,IAAA;AAAA,YACT;AAAA,WACF;AAEA,UAAA,SAAA,CAAU,KAAK,QAAQ,CAAA;AAAA,QACzB,SAAS,KAAA,EAAO;AACd,UAAA,MAAA,CAAO,IAAA;AAAA,YACL,qDAAqD,IAAA,CAAK,IAAI,oBAAoB,UAAA,CAAW,IAAI,aAAa,KAAK,CAAA,+BAAA;AAAA,WACrH;AAAA,QAEF;AAAA,MACF;AAGA,MAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,MAAA,sBAAA,IAA0B,SAAA,CAAU,MAAA;AAEpC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,MAAA,EAAS,WAAW,CAAA,CAAA,EAAI,YAAY,eAAe,SAAA,CAAU,MAAM,CAAA,iDAAA,EAAoD,UAAA,CAAW,IAAI,CAAA;AAAA,OACxI;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,sBAAA,EAAuB;AAAA,EAClC,CAAA;AAMA,EAAA,MAAM,uBAAA,GAA0B,OAC9B,kBAAA,KACG;AACH,IAAA,MAAM,gBAAA,GAAmB,MAAM,kBAAA,CAAmB,QAAA,EAAS;AAE3D,IAAA,IAAI,gBAAA,CAAiB,WAAW,CAAA,EAAG;AACjC,MAAA,MAAA,CAAO,KAAK,mDAAmD,CAAA;AAC/D,MAAA;AAAA,IACF;AAGA,IAAA,IAAI,oBAAA,GAAuB,gBAAA;AAG3B,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,iGAAA,EAAoG,eAAA,CACjG,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,oBAAA,GAAuB,oBAAA,CAAqB,OAAO,CAAA,IAAA,KAAQ;AACzD,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAAA,IACH;AAGA,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,6FAAA,EAAgG,eAAA,CAC7F,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,MAAM,aAAA,GAAgB,oBAAA,CAAqB,MAAA,CAAO,CAAA,IAAA,KAAQ;AACxD,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAED,MAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,wBAAA,EAA2B,cACxB,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CACf,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AACA,MAAA,oBAAA,GAAuB,oBAAA,CAAqB,OAAO,CAAA,IAAA,KAAQ;AACzD,QAAA,OAAO,CAAC,eAAA,CAAgB,IAAA;AAAA,UAAK,CAAA,OAAA,KAC3B,OAAA,CAAQ,KAAA,CAAO,IAAA,CAAK,KAAK,IAAK;AAAA,SAChC;AAAA,MACF,CAAC,CAAA;AAAA,IACH;AAEA,IAAA,IAAI,oBAAA,CAAqB,WAAW,CAAA,EAAG;AACrC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL;AAAA,OACF;AACA,MAAA;AAAA,IACF;AAEA,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,wBAAA,EAA2B,qBACxB,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CACf,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAEA,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,UAAA,EAAa,qBAAqB,MAAM,CAAA,+BAAA;AAAA,KAC1C;AAGA,IAAA,KAAA,MAAW,QAAQ,oBAAA,EAAsB;AACvC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,oCAAA,EAAuC,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,OAC9D;AAGA,MAAA,MAAM,kBAAkB,eAAA,CAAgB,IAAA;AAAA,QAAK,CAAA,OAAA,KAC3C,OAAA,CAAQ,KAAA,CAAO,IAAA,CAAK,KAAK,IAAK;AAAA,OAChC;AAGA,MAAA,MAAM,yBAAA,GAA4B,iBAAiB,SAAA,IAAa,SAAA;AAGhE,MAAA,MAAM,wBAAA,GACJ,iBAAiB,cAAA,IAAkB,oBAAA;AAErC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,wCACE,IAAA,CAAK,IACP,MAAM,yBAAA,CAA0B,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,OAC5C;AAEA,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,wCACE,IAAA,CAAK,IACP,MAAM,wBAAA,CAAyB,IAAA,CAAK,IAAI,CAAC,CAAA,CAAA;AAAA,OAC3C;AAGA,MAAA,IAAI,KAAA,GAAQ,MAAM,kBAAA,CAAmB,YAAA;AAAA,QACnC,IAAA,CAAK,EAAA;AAAA,QACL;AAAA,OACF;AAGA,MAAA,MAAM,aAAaC,iDAAA,CAAiB;AAAA,QAClC,iBAAA,EAAmB;AAAA,OACpB,CAAA;AAED,MAAA,MAAM,oBAAoB,KAAA,CAAM,MAAA;AAGhC,MAAA,MAAM,gBAAgB,KAAA,CAAM,MAAA;AAAA,QAC1B,UAAQ,IAAA,CAAK,IAAA,IAAQ,UAAA,CAAW,iBAAA,CAAkB,KAAK,IAAI;AAAA,OAC7D;AAEA,MAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,+BAAA,EAAkC,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,aAAA,CAC7C,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAE,IAAI,CAAA,CACf,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SACf;AAAA,MACF;AAEA,MAAA,KAAA,GAAQ,UAAA,CAAW,YAAY,KAAK,CAAA;AACpC,MAAA,MAAM,iBAAA,GAAoB,oBAAoB,KAAA,CAAM,MAAA;AAEpD,MAAA,IAAI,oBAAoB,CAAA,EAAG;AACzB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,aAAA,EAAgB,iBAAiB,CAAA,uBAAA,EAA0B,IAAA,CAAK,IAAI,CAAA,iCAAA;AAAA,SACtE;AAAA,MACF;AAEA,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,+DACE,IAAA,CAAK,IACP,KACE,IAAA,CAAK,EACP,4CAA4C,yBAAA,CAA0B,IAAA;AAAA,YACpE;AAAA,WACD,CAAA,CAAA;AAAA,SACH;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,EAAE,sBAAA,EAAuB,GAAI,MAAM,qBAAA,CAAsB;AAAA,QAC7D,UAAA,EAAY,IAAA;AAAA,QACZ,KAAA;AAAA,QACA;AAAA,OACD,CAAA;AAED,MAAA,IAAI,2BAA2B,CAAA,EAAG;AAChC,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,mFAAA,EAAsF,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,SAC7G;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,gCAAA,EAAmC,sBAAsB,CAAA,8EAAA,EAAiF,IAAA,CAAK,IAAI,CAAA;AAAA,OACrJ;AAAA,IACF;AAAA,EACF,CAAA;AAEA,EAAA,OAAO,EAAE,uBAAA,EAAwB;AACnC;;;;"}
|
|
@@ -83,41 +83,50 @@ const createWikiIngestor = async ({
|
|
|
83
83
|
for (let index = 0; index < pagesBatch.length; index++) {
|
|
84
84
|
const page = pagesBatch[index];
|
|
85
85
|
const globalIndex = batchStart + index;
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
86
|
+
try {
|
|
87
|
+
const [content, lastUpdated] = await Promise.all([
|
|
88
|
+
azureDevOpsService.getWikiPageContent(wiki.id, page.path),
|
|
89
|
+
azureDevOpsService.getWikiPageLastUpdated(wiki.id, page.path)
|
|
90
|
+
]);
|
|
91
|
+
const completionStats = backstagePluginAiAssistantCommon.getProgressStats(
|
|
92
|
+
globalIndex + 1,
|
|
93
|
+
pages.length
|
|
94
|
+
);
|
|
95
|
+
logger.info(
|
|
96
|
+
`Retrieved content for Azure DevOps page: "${page.path}" in wiki: "${wiki.name}" [Progress: ${completionStats.completed}/${completionStats.total} (${completionStats.percentage}%) completed of wiki]`
|
|
97
|
+
);
|
|
98
|
+
const pageContent = await backstagePluginAiAssistantNode.streamToString(content);
|
|
99
|
+
logger.debug(
|
|
100
|
+
`Raw response for page "${page.path}" (length: ${pageContent.length})`
|
|
101
|
+
);
|
|
102
|
+
const pageUrl = page.remoteUrl || page.url;
|
|
103
|
+
if (!pageContent || pageContent.trim().length === 0) {
|
|
104
|
+
logger.warn(
|
|
105
|
+
`No content found for Azure DevOps page: "${page.path}" in wiki: "${wiki.name}". Skipping.`
|
|
106
|
+
);
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
const document = {
|
|
110
|
+
metadata: {
|
|
111
|
+
source: module$1.MODULE_ID,
|
|
112
|
+
id: `${wiki.id}:${page.path}`,
|
|
113
|
+
url: pageUrl,
|
|
114
|
+
organization: azureDevOpsService.organization,
|
|
115
|
+
project: azureDevOpsService.project,
|
|
116
|
+
wiki: wiki.name
|
|
117
|
+
},
|
|
118
|
+
content: pageContent,
|
|
119
|
+
lastUpdated
|
|
120
|
+
};
|
|
121
|
+
logger.debug(
|
|
122
|
+
`Created embedding document for Azure DevOps page: "${page.path}" in wiki: "${wiki.name}" content length: "${document.content.length}", page url: "${document.metadata.url}"`
|
|
123
|
+
);
|
|
124
|
+
documents.push(document);
|
|
125
|
+
} catch (error) {
|
|
100
126
|
logger.warn(
|
|
101
|
-
`
|
|
127
|
+
`Failed to retrieve content for Azure DevOps page: "${page.path}" in wiki: "${wiki.name}". Error: ${error}. Skipping page and continuing.`
|
|
102
128
|
);
|
|
103
|
-
continue;
|
|
104
129
|
}
|
|
105
|
-
const document = {
|
|
106
|
-
metadata: {
|
|
107
|
-
source: module$1.MODULE_ID,
|
|
108
|
-
id: `${wiki.id}:${page.path}`,
|
|
109
|
-
url: pageUrl,
|
|
110
|
-
organization: azureDevOpsService.organization,
|
|
111
|
-
project: azureDevOpsService.project,
|
|
112
|
-
wiki: wiki.name
|
|
113
|
-
},
|
|
114
|
-
content: pageContent,
|
|
115
|
-
lastUpdated
|
|
116
|
-
};
|
|
117
|
-
logger.debug(
|
|
118
|
-
`Created embedding document for Azure DevOps page: "${page.path}" in wiki: "${wiki.name}" content length: "${document.content.length}", page url: "${document.metadata.url}"`
|
|
119
|
-
);
|
|
120
|
-
documents.push(document);
|
|
121
130
|
}
|
|
122
131
|
await saveDocumentsBatch(documents);
|
|
123
132
|
totalDocumentsIngested += documents.length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wiki.cjs.js","sources":["../../../src/services/ingestor/wiki.ts"],"sourcesContent":["import {\n LoggerService,\n RootConfigService,\n} from '@backstage/backend-plugin-api';\nimport {\n EmbeddingDocument,\n IngestorOptions,\n streamToString,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { AzureDevOpsService } from '../azure-devops';\nimport { Config } from '../../../config';\nimport { MODULE_ID } from '../../constants/module';\nimport { getProgressStats } from '@sweetoburrito/backstage-plugin-ai-assistant-common';\nimport { DEFAULT_WIKI_PAGE_BATCH_SIZE } from '../../constants/default-wiki-page-batch-size';\nimport {\n WikiPage,\n WikiV2,\n} from 'azure-devops-node-api/interfaces/WikiInterfaces';\n\ntype WikiIngestorOptions = {\n config: RootConfigService;\n logger: LoggerService;\n azureDevOpsService: AzureDevOpsService;\n};\n\nexport const createWikiIngestor = async ({\n config,\n logger,\n azureDevOpsService,\n}: WikiIngestorOptions) => {\n // Get configuration values\n const wikisFilter = config.getOptional<\n Config['aiAssistant']['ingestors']['azureDevOps']['wikis']\n >('aiAssistant.ingestors.azureDevOps.wikis');\n\n // Process and validate wiki filters\n type WikiMatcher = {\n value: string;\n regex: RegExp;\n };\n\n const includeMatchers: WikiMatcher[] = [];\n const excludeMatchers: WikiMatcher[] = [];\n\n if (wikisFilter?.include) {\n for (const filter of wikisFilter.include) {\n try {\n // All strings are treated as regex patterns; escape special characters for exact literal matches\n const regex = new RegExp(filter.name);\n includeMatchers.push({\n value: filter.name,\n regex,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in wiki include '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid wiki include pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (wikisFilter?.exclude) {\n for (const filter of wikisFilter.exclude) {\n try {\n // All strings are valid regex - plain strings match exactly, patterns match as regex\n const regex = new RegExp(filter.name);\n excludeMatchers.push({\n value: filter.name,\n regex,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in wiki exclude '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid wiki exclude pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (includeMatchers.length > 0) {\n logger.info(\n `Wiki include filters: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n if (excludeMatchers.length > 0) {\n logger.info(\n `Wiki exclude filters: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n\n // Get batch size for processing pages (default to 50 pages per batch)\n const pagesBatchSize =\n config.getOptionalNumber(\n 'aiAssistant.ingestors.azureDevOps.pagesBatchSize',\n ) ?? DEFAULT_WIKI_PAGE_BATCH_SIZE;\n\n /** Ingest Azure DevOps wiki pages in batches\n * @param wiki - The wiki to ingest pages from\n * @param pages - The list of pages to ingest from the wiki\n * @param saveDocumentsBatch - Function to save a batch of embedding documents\n * @returns Total number of documents ingested and sent for embedding from the wiki\n */\n const ingestWikiByPageBatch = async ({\n wiki,\n pages,\n saveDocumentsBatch,\n }: {\n wiki: WikiV2;\n pages: WikiPage[];\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'];\n }) => {\n logger.info(\n `Processing ${pages.length} pages from wiki \"${wiki.name}\" in batches of ${pagesBatchSize}`,\n );\n\n let totalDocumentsIngested = 0;\n\n // Process pages in batches to manage memory and performance\n\n // Calculate total number of batches\n const totalBatches = Math.ceil(pages.length / pagesBatchSize);\n\n // Process each batch\n for (\n let batchStart = 0;\n batchStart < pages.length;\n batchStart += pagesBatchSize\n ) {\n const batchEnd = Math.min(batchStart + pagesBatchSize, pages.length);\n const pagesBatch = pages.slice(batchStart, batchEnd);\n const batchNumber = Math.floor(batchStart / pagesBatchSize) + 1;\n\n logger.info(\n `Processing batch ${batchNumber}/${totalBatches} (${pagesBatch.length} pages) for wiki \"${wiki.name}\"`,\n );\n\n // Generate embedding documents for each page in the current batch\n const documents: EmbeddingDocument[] = [];\n\n for (let index = 0; index < pagesBatch.length; index++) {\n const page = pagesBatch[index];\n const globalIndex = batchStart + index;\n\n const [content, lastUpdated] = await Promise.all([\n azureDevOpsService.getWikiPageContent(wiki.id!, page.path!),\n azureDevOpsService.getWikiPageLastUpdated(wiki.id!, page.path!),\n ]);\n\n const completionStats = getProgressStats(globalIndex + 1, pages.length);\n\n logger.info(\n `Retrieved content for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\" [Progress: ${completionStats.completed}/${completionStats.total} (${completionStats.percentage}%) completed of wiki]`,\n );\n\n // The API returns plain markdown text directly\n const pageContent = await streamToString(content);\n\n logger.debug(\n `Raw response for page \"${page.path}\" (length: ${pageContent.length})`,\n );\n // Use remoteUrl which points to the user-facing wiki page, not the API endpoint\n const pageUrl = page.remoteUrl || page.url!;\n\n // Check if we have actual content (not empty or just whitespace)\n if (!pageContent || pageContent.trim().length === 0) {\n logger.warn(\n `No content found for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\". Skipping.`,\n );\n continue;\n }\n\n const document: EmbeddingDocument = {\n metadata: {\n source: MODULE_ID,\n id: `${wiki.id}:${page.path}`,\n url: pageUrl,\n organization: azureDevOpsService.organization,\n project: azureDevOpsService.project,\n wiki: wiki.name!,\n },\n content: pageContent,\n lastUpdated,\n };\n\n logger.debug(\n `Created embedding document for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\" content length: \"${document.content.length}\", page url: \"${document.metadata.url}\"`,\n );\n\n documents.push(document);\n }\n\n // Save the current batch of documents\n await saveDocumentsBatch(documents);\n\n totalDocumentsIngested += documents.length;\n\n logger.info(\n `Batch ${batchNumber}/${totalBatches} completed: ${documents.length} documents ingested for Azure DevOps wiki: ${wiki.name}`,\n );\n }\n\n return { totalDocumentsIngested };\n };\n\n /** Ingest Azure DevOps wikis in batches */\n const ingestWikisBatch = async (\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n ) => {\n const wikisList = await azureDevOpsService.getWikis();\n\n if (wikisList.length === 0) {\n logger.warn('No wikis found in the Azure DevOps project');\n return;\n }\n\n // Filter wikis using matchers\n let wikisToIngest = wikisList;\n\n // If include matchers exist, only include wikis that match at least one\n if (includeMatchers.length > 0) {\n logger.info(\n `Include filter found. Only including wikis matching the following patterns for ingestion: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n wikisToIngest = wikisToIngest.filter(wiki => {\n return includeMatchers.some(matcher => matcher.regex!.test(wiki.name!));\n });\n }\n\n // Apply exclusions\n if (excludeMatchers.length > 0) {\n logger.info(\n `Exclude filter found. Excluding wikis matching the following patterns from ingestion: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n const excludedWikis = wikisToIngest.filter(wiki => {\n return excludeMatchers.some(matcher => matcher.regex!.test(wiki.name!));\n });\n\n if (excludedWikis.length > 0) {\n logger.info(\n `Excluding wikis: ${excludedWikis.map(w => w.name).join(', ')}`,\n );\n }\n wikisToIngest = wikisToIngest.filter(wiki => {\n return !excludeMatchers.some(matcher =>\n matcher.regex!.test(wiki.name!),\n );\n });\n }\n\n if (wikisToIngest.length === 0) {\n logger.warn('No wikis found for ingestion after applying the filter');\n return;\n }\n\n logger.debug(\n `Wikis to ingest: ${wikisToIngest.map(w => w.name).join(', ')}`,\n );\n\n logger.info(`Ingesting ${wikisToIngest.length} wikis from Azure DevOps`);\n\n // Get items from each wiki and create documents to be embedded\n for (const wiki of wikisToIngest) {\n logger.info(`Beginning ingestion for wiki: ${wiki.name} (${wiki.id})`);\n\n // Get the pages to be ingested from the wiki based on the file types filter\n const pages = await azureDevOpsService.getWikiPages(wiki.id!);\n\n if (pages.length === 0) {\n logger.warn(\n `No pages found for ingestion in the Azure DevOps wiki ${wiki.name} (${wiki.id})`,\n );\n continue;\n }\n\n const { totalDocumentsIngested } = await ingestWikiByPageBatch({\n wiki,\n pages,\n saveDocumentsBatch,\n });\n\n if (totalDocumentsIngested === 0) {\n logger.warn(\n `No documents were ingested and sent for embedding from the Azure DevOps wiki ${wiki.name} (${wiki.id})`,\n );\n continue;\n }\n\n logger.info(\n `Wiki ingestion completed: ${totalDocumentsIngested} total documents ingested and sent for embedding for Azure DevOps wiki: ${wiki.name}`,\n );\n }\n };\n\n return { ingestWikisBatch };\n};\n"],"names":["DEFAULT_WIKI_PAGE_BATCH_SIZE","getProgressStats","streamToString","MODULE_ID"],"mappings":";;;;;;;AAyBO,MAAM,qBAAqB,OAAO;AAAA,EACvC,MAAA;AAAA,EACA,MAAA;AAAA,EACA;AACF,CAAA,KAA2B;AAEzB,EAAA,MAAM,WAAA,GAAc,MAAA,CAAO,WAAA,CAEzB,yCAAyC,CAAA;AAQ3C,EAAA,MAAM,kBAAiC,EAAC;AACxC,EAAA,MAAM,kBAAiC,EAAC;AAExC,EAAA,IAAI,aAAa,OAAA,EAAS;AACxB,IAAA,KAAA,MAAW,MAAA,IAAU,YAAY,OAAA,EAAS;AACxC,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd;AAAA,SACD,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,4CAAA,EAA+C,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACvE;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,8BAAA,EAAiC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,aAAa,OAAA,EAAS;AACxB,IAAA,KAAA,MAAW,MAAA,IAAU,YAAY,OAAA,EAAS;AACxC,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd;AAAA,SACD,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,4CAAA,EAA+C,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACvE;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,8BAAA,EAAiC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,sBAAA,EAAyB,eAAA,CACtB,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AACA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,sBAAA,EAAyB,eAAA,CACtB,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AAGA,EAAA,MAAM,iBACJ,MAAA,CAAO,iBAAA;AAAA,IACL;AAAA,GACF,IAAKA,qDAAA;AAQP,EAAA,MAAM,wBAAwB,OAAO;AAAA,IACnC,IAAA;AAAA,IACA,KAAA;AAAA,IACA;AAAA,GACF,KAIM;AACJ,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,cAAc,KAAA,CAAM,MAAM,qBAAqB,IAAA,CAAK,IAAI,mBAAmB,cAAc,CAAA;AAAA,KAC3F;AAEA,IAAA,IAAI,sBAAA,GAAyB,CAAA;AAK7B,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,SAAS,cAAc,CAAA;AAG5D,IAAA,KAAA,IACM,aAAa,CAAA,EACjB,UAAA,GAAa,KAAA,CAAM,MAAA,EACnB,cAAc,cAAA,EACd;AACA,MAAA,MAAM,WAAW,IAAA,CAAK,GAAA,CAAI,UAAA,GAAa,cAAA,EAAgB,MAAM,MAAM,CAAA;AACnE,MAAA,MAAM,UAAA,GAAa,KAAA,CAAM,KAAA,CAAM,UAAA,EAAY,QAAQ,CAAA;AACnD,MAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,UAAA,GAAa,cAAc,CAAA,GAAI,CAAA;AAE9D,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,iBAAA,EAAoB,WAAW,CAAA,CAAA,EAAI,YAAY,KAAK,UAAA,CAAW,MAAM,CAAA,kBAAA,EAAqB,IAAA,CAAK,IAAI,CAAA,CAAA;AAAA,OACrG;AAGA,MAAA,MAAM,YAAiC,EAAC;AAExC,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,UAAA,CAAW,QAAQ,KAAA,EAAA,EAAS;AACtD,QAAA,MAAM,IAAA,GAAO,WAAW,KAAK,CAAA;AAC7B,QAAA,MAAM,cAAc,UAAA,GAAa,KAAA;AAEjC,QAAA,MAAM,CAAC,OAAA,EAAS,WAAW,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,UAC/C,kBAAA,CAAmB,kBAAA,CAAmB,IAAA,CAAK,EAAA,EAAK,KAAK,IAAK,CAAA;AAAA,UAC1D,kBAAA,CAAmB,sBAAA,CAAuB,IAAA,CAAK,EAAA,EAAK,KAAK,IAAK;AAAA,SAC/D,CAAA;AAED,QAAA,MAAM,eAAA,GAAkBC,iDAAA,CAAiB,WAAA,GAAc,CAAA,EAAG,MAAM,MAAM,CAAA;AAEtE,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,0CAAA,EAA6C,IAAA,CAAK,IAAI,CAAA,YAAA,EAAe,KAAK,IAAI,CAAA,aAAA,EAAgB,eAAA,CAAgB,SAAS,CAAA,CAAA,EAAI,eAAA,CAAgB,KAAK,CAAA,EAAA,EAAK,gBAAgB,UAAU,CAAA,qBAAA;AAAA,SACjL;AAGA,QAAA,MAAM,WAAA,GAAc,MAAMC,6CAAA,CAAe,OAAO,CAAA;AAEhD,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,uBAAA,EAA0B,IAAA,CAAK,IAAI,CAAA,WAAA,EAAc,YAAY,MAAM,CAAA,CAAA;AAAA,SACrE;AAEA,QAAA,MAAM,OAAA,GAAU,IAAA,CAAK,SAAA,IAAa,IAAA,CAAK,GAAA;AAGvC,QAAA,IAAI,CAAC,WAAA,IAAe,WAAA,CAAY,IAAA,EAAK,CAAE,WAAW,CAAA,EAAG;AACnD,UAAA,MAAA,CAAO,IAAA;AAAA,YACL,CAAA,yCAAA,EAA4C,IAAA,CAAK,IAAI,CAAA,YAAA,EAAe,KAAK,IAAI,CAAA,YAAA;AAAA,WAC/E;AACA,UAAA;AAAA,QACF;AAEA,QAAA,MAAM,QAAA,GAA8B;AAAA,UAClC,QAAA,EAAU;AAAA,YACR,MAAA,EAAQC,kBAAA;AAAA,YACR,IAAI,CAAA,EAAG,IAAA,CAAK,EAAE,CAAA,CAAA,EAAI,KAAK,IAAI,CAAA,CAAA;AAAA,YAC3B,GAAA,EAAK,OAAA;AAAA,YACL,cAAc,kBAAA,CAAmB,YAAA;AAAA,YACjC,SAAS,kBAAA,CAAmB,OAAA;AAAA,YAC5B,MAAM,IAAA,CAAK;AAAA,WACb;AAAA,UACA,OAAA,EAAS,WAAA;AAAA,UACT;AAAA,SACF;AAEA,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,mDAAA,EAAsD,IAAA,CAAK,IAAI,CAAA,YAAA,EAAe,IAAA,CAAK,IAAI,CAAA,mBAAA,EAAsB,QAAA,CAAS,OAAA,CAAQ,MAAM,CAAA,cAAA,EAAiB,QAAA,CAAS,SAAS,GAAG,CAAA,CAAA;AAAA,SAC5K;AAEA,QAAA,SAAA,CAAU,KAAK,QAAQ,CAAA;AAAA,MACzB;AAGA,MAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,MAAA,sBAAA,IAA0B,SAAA,CAAU,MAAA;AAEpC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,MAAA,EAAS,WAAW,CAAA,CAAA,EAAI,YAAY,eAAe,SAAA,CAAU,MAAM,CAAA,2CAAA,EAA8C,IAAA,CAAK,IAAI,CAAA;AAAA,OAC5H;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,sBAAA,EAAuB;AAAA,EAClC,CAAA;AAGA,EAAA,MAAM,gBAAA,GAAmB,OACvB,kBAAA,KACG;AACH,IAAA,MAAM,SAAA,GAAY,MAAM,kBAAA,CAAmB,QAAA,EAAS;AAEpD,IAAA,IAAI,SAAA,CAAU,WAAW,CAAA,EAAG;AAC1B,MAAA,MAAA,CAAO,KAAK,4CAA4C,CAAA;AACxD,MAAA;AAAA,IACF;AAGA,IAAA,IAAI,aAAA,GAAgB,SAAA;AAGpB,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,0FAAA,EAA6F,eAAA,CAC1F,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,aAAA,GAAgB,aAAA,CAAc,OAAO,CAAA,IAAA,KAAQ;AAC3C,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAAA,IACH;AAGA,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,sFAAA,EAAyF,eAAA,CACtF,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,MAAM,aAAA,GAAgB,aAAA,CAAc,MAAA,CAAO,CAAA,IAAA,KAAQ;AACjD,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAED,MAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,iBAAA,EAAoB,cAAc,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SAC/D;AAAA,MACF;AACA,MAAA,aAAA,GAAgB,aAAA,CAAc,OAAO,CAAA,IAAA,KAAQ;AAC3C,QAAA,OAAO,CAAC,eAAA,CAAgB,IAAA;AAAA,UAAK,CAAA,OAAA,KAC3B,OAAA,CAAQ,KAAA,CAAO,IAAA,CAAK,KAAK,IAAK;AAAA,SAChC;AAAA,MACF,CAAC,CAAA;AAAA,IACH;AAEA,IAAA,IAAI,aAAA,CAAc,WAAW,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,KAAK,wDAAwD,CAAA;AACpE,MAAA;AAAA,IACF;AAEA,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,iBAAA,EAAoB,cAAc,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KAC/D;AAEA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,UAAA,EAAa,aAAA,CAAc,MAAM,CAAA,wBAAA,CAA0B,CAAA;AAGvE,IAAA,KAAA,MAAW,QAAQ,aAAA,EAAe;AAChC,MAAA,MAAA,CAAO,KAAK,CAAA,8BAAA,EAAiC,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,IAAA,CAAK,EAAE,CAAA,CAAA,CAAG,CAAA;AAGrE,MAAA,MAAM,KAAA,GAAQ,MAAM,kBAAA,CAAmB,YAAA,CAAa,KAAK,EAAG,CAAA;AAE5D,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,sDAAA,EAAyD,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,SAChF;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,EAAE,sBAAA,EAAuB,GAAI,MAAM,qBAAA,CAAsB;AAAA,QAC7D,IAAA;AAAA,QACA,KAAA;AAAA,QACA;AAAA,OACD,CAAA;AAED,MAAA,IAAI,2BAA2B,CAAA,EAAG;AAChC,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,6EAAA,EAAgF,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,SACvG;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,0BAAA,EAA6B,sBAAsB,CAAA,wEAAA,EAA2E,IAAA,CAAK,IAAI,CAAA;AAAA,OACzI;AAAA,IACF;AAAA,EACF,CAAA;AAEA,EAAA,OAAO,EAAE,gBAAA,EAAiB;AAC5B;;;;"}
|
|
1
|
+
{"version":3,"file":"wiki.cjs.js","sources":["../../../src/services/ingestor/wiki.ts"],"sourcesContent":["import {\n LoggerService,\n RootConfigService,\n} from '@backstage/backend-plugin-api';\nimport {\n EmbeddingDocument,\n IngestorOptions,\n streamToString,\n} from '@sweetoburrito/backstage-plugin-ai-assistant-node';\nimport { AzureDevOpsService } from '../azure-devops';\nimport { Config } from '../../../config';\nimport { MODULE_ID } from '../../constants/module';\nimport { getProgressStats } from '@sweetoburrito/backstage-plugin-ai-assistant-common';\nimport { DEFAULT_WIKI_PAGE_BATCH_SIZE } from '../../constants/default-wiki-page-batch-size';\nimport {\n WikiPage,\n WikiV2,\n} from 'azure-devops-node-api/interfaces/WikiInterfaces';\n\ntype WikiIngestorOptions = {\n config: RootConfigService;\n logger: LoggerService;\n azureDevOpsService: AzureDevOpsService;\n};\n\nexport const createWikiIngestor = async ({\n config,\n logger,\n azureDevOpsService,\n}: WikiIngestorOptions) => {\n // Get configuration values\n const wikisFilter = config.getOptional<\n Config['aiAssistant']['ingestors']['azureDevOps']['wikis']\n >('aiAssistant.ingestors.azureDevOps.wikis');\n\n // Process and validate wiki filters\n type WikiMatcher = {\n value: string;\n regex: RegExp;\n };\n\n const includeMatchers: WikiMatcher[] = [];\n const excludeMatchers: WikiMatcher[] = [];\n\n if (wikisFilter?.include) {\n for (const filter of wikisFilter.include) {\n try {\n // All strings are treated as regex patterns; escape special characters for exact literal matches\n const regex = new RegExp(filter.name);\n includeMatchers.push({\n value: filter.name,\n regex,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in wiki include '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid wiki include pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (wikisFilter?.exclude) {\n for (const filter of wikisFilter.exclude) {\n try {\n // All strings are valid regex - plain strings match exactly, patterns match as regex\n const regex = new RegExp(filter.name);\n excludeMatchers.push({\n value: filter.name,\n regex,\n });\n } catch (error) {\n logger.error(\n `Invalid regular expression in wiki exclude '${filter.name}': ${error}`,\n );\n throw new Error(\n `Invalid wiki exclude pattern '${filter.name}': ${error}`,\n );\n }\n }\n }\n\n if (includeMatchers.length > 0) {\n logger.info(\n `Wiki include filters: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n if (excludeMatchers.length > 0) {\n logger.info(\n `Wiki exclude filters: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n }\n\n // Get batch size for processing pages (default to 50 pages per batch)\n const pagesBatchSize =\n config.getOptionalNumber(\n 'aiAssistant.ingestors.azureDevOps.pagesBatchSize',\n ) ?? DEFAULT_WIKI_PAGE_BATCH_SIZE;\n\n /** Ingest Azure DevOps wiki pages in batches\n * @param wiki - The wiki to ingest pages from\n * @param pages - The list of pages to ingest from the wiki\n * @param saveDocumentsBatch - Function to save a batch of embedding documents\n * @returns Total number of documents ingested and sent for embedding from the wiki\n */\n const ingestWikiByPageBatch = async ({\n wiki,\n pages,\n saveDocumentsBatch,\n }: {\n wiki: WikiV2;\n pages: WikiPage[];\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'];\n }) => {\n logger.info(\n `Processing ${pages.length} pages from wiki \"${wiki.name}\" in batches of ${pagesBatchSize}`,\n );\n\n let totalDocumentsIngested = 0;\n\n // Process pages in batches to manage memory and performance\n\n // Calculate total number of batches\n const totalBatches = Math.ceil(pages.length / pagesBatchSize);\n\n // Process each batch\n for (\n let batchStart = 0;\n batchStart < pages.length;\n batchStart += pagesBatchSize\n ) {\n const batchEnd = Math.min(batchStart + pagesBatchSize, pages.length);\n const pagesBatch = pages.slice(batchStart, batchEnd);\n const batchNumber = Math.floor(batchStart / pagesBatchSize) + 1;\n\n logger.info(\n `Processing batch ${batchNumber}/${totalBatches} (${pagesBatch.length} pages) for wiki \"${wiki.name}\"`,\n );\n\n // Generate embedding documents for each page in the current batch\n const documents: EmbeddingDocument[] = [];\n\n for (let index = 0; index < pagesBatch.length; index++) {\n const page = pagesBatch[index];\n const globalIndex = batchStart + index;\n\n try {\n const [content, lastUpdated] = await Promise.all([\n azureDevOpsService.getWikiPageContent(wiki.id!, page.path!),\n azureDevOpsService.getWikiPageLastUpdated(wiki.id!, page.path!),\n ]);\n\n const completionStats = getProgressStats(\n globalIndex + 1,\n pages.length,\n );\n\n logger.info(\n `Retrieved content for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\" [Progress: ${completionStats.completed}/${completionStats.total} (${completionStats.percentage}%) completed of wiki]`,\n );\n\n // The API returns plain markdown text directly\n const pageContent = await streamToString(content);\n\n logger.debug(\n `Raw response for page \"${page.path}\" (length: ${pageContent.length})`,\n );\n // Use remoteUrl which points to the user-facing wiki page, not the API endpoint\n const pageUrl = page.remoteUrl || page.url!;\n\n // Check if we have actual content (not empty or just whitespace)\n if (!pageContent || pageContent.trim().length === 0) {\n logger.warn(\n `No content found for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\". Skipping.`,\n );\n continue;\n }\n\n const document: EmbeddingDocument = {\n metadata: {\n source: MODULE_ID,\n id: `${wiki.id}:${page.path}`,\n url: pageUrl,\n organization: azureDevOpsService.organization,\n project: azureDevOpsService.project,\n wiki: wiki.name!,\n },\n content: pageContent,\n lastUpdated,\n };\n\n logger.debug(\n `Created embedding document for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\" content length: \"${document.content.length}\", page url: \"${document.metadata.url}\"`,\n );\n\n documents.push(document);\n } catch (error) {\n logger.warn(\n `Failed to retrieve content for Azure DevOps page: \"${page.path}\" in wiki: \"${wiki.name}\". Error: ${error}. Skipping page and continuing.`,\n );\n // Continue to next page instead of crashing entire ingestion\n }\n }\n\n // Save the current batch of documents\n await saveDocumentsBatch(documents);\n\n totalDocumentsIngested += documents.length;\n\n logger.info(\n `Batch ${batchNumber}/${totalBatches} completed: ${documents.length} documents ingested for Azure DevOps wiki: ${wiki.name}`,\n );\n }\n\n return { totalDocumentsIngested };\n };\n\n /** Ingest Azure DevOps wikis in batches */\n const ingestWikisBatch = async (\n saveDocumentsBatch: IngestorOptions['saveDocumentsBatch'],\n ) => {\n const wikisList = await azureDevOpsService.getWikis();\n\n if (wikisList.length === 0) {\n logger.warn('No wikis found in the Azure DevOps project');\n return;\n }\n\n // Filter wikis using matchers\n let wikisToIngest = wikisList;\n\n // If include matchers exist, only include wikis that match at least one\n if (includeMatchers.length > 0) {\n logger.info(\n `Include filter found. Only including wikis matching the following patterns for ingestion: ${includeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n wikisToIngest = wikisToIngest.filter(wiki => {\n return includeMatchers.some(matcher => matcher.regex!.test(wiki.name!));\n });\n }\n\n // Apply exclusions\n if (excludeMatchers.length > 0) {\n logger.info(\n `Exclude filter found. Excluding wikis matching the following patterns from ingestion: ${excludeMatchers\n .map(m => `'${m.value}'`)\n .join(', ')}`,\n );\n\n const excludedWikis = wikisToIngest.filter(wiki => {\n return excludeMatchers.some(matcher => matcher.regex!.test(wiki.name!));\n });\n\n if (excludedWikis.length > 0) {\n logger.info(\n `Excluding wikis: ${excludedWikis.map(w => w.name).join(', ')}`,\n );\n }\n wikisToIngest = wikisToIngest.filter(wiki => {\n return !excludeMatchers.some(matcher =>\n matcher.regex!.test(wiki.name!),\n );\n });\n }\n\n if (wikisToIngest.length === 0) {\n logger.warn('No wikis found for ingestion after applying the filter');\n return;\n }\n\n logger.debug(\n `Wikis to ingest: ${wikisToIngest.map(w => w.name).join(', ')}`,\n );\n\n logger.info(`Ingesting ${wikisToIngest.length} wikis from Azure DevOps`);\n\n // Get items from each wiki and create documents to be embedded\n for (const wiki of wikisToIngest) {\n logger.info(`Beginning ingestion for wiki: ${wiki.name} (${wiki.id})`);\n\n // Get the pages to be ingested from the wiki based on the file types filter\n const pages = await azureDevOpsService.getWikiPages(wiki.id!);\n\n if (pages.length === 0) {\n logger.warn(\n `No pages found for ingestion in the Azure DevOps wiki ${wiki.name} (${wiki.id})`,\n );\n continue;\n }\n\n const { totalDocumentsIngested } = await ingestWikiByPageBatch({\n wiki,\n pages,\n saveDocumentsBatch,\n });\n\n if (totalDocumentsIngested === 0) {\n logger.warn(\n `No documents were ingested and sent for embedding from the Azure DevOps wiki ${wiki.name} (${wiki.id})`,\n );\n continue;\n }\n\n logger.info(\n `Wiki ingestion completed: ${totalDocumentsIngested} total documents ingested and sent for embedding for Azure DevOps wiki: ${wiki.name}`,\n );\n }\n };\n\n return { ingestWikisBatch };\n};\n"],"names":["DEFAULT_WIKI_PAGE_BATCH_SIZE","getProgressStats","streamToString","MODULE_ID"],"mappings":";;;;;;;AAyBO,MAAM,qBAAqB,OAAO;AAAA,EACvC,MAAA;AAAA,EACA,MAAA;AAAA,EACA;AACF,CAAA,KAA2B;AAEzB,EAAA,MAAM,WAAA,GAAc,MAAA,CAAO,WAAA,CAEzB,yCAAyC,CAAA;AAQ3C,EAAA,MAAM,kBAAiC,EAAC;AACxC,EAAA,MAAM,kBAAiC,EAAC;AAExC,EAAA,IAAI,aAAa,OAAA,EAAS;AACxB,IAAA,KAAA,MAAW,MAAA,IAAU,YAAY,OAAA,EAAS;AACxC,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd;AAAA,SACD,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,4CAAA,EAA+C,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACvE;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,8BAAA,EAAiC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,aAAa,OAAA,EAAS;AACxB,IAAA,KAAA,MAAW,MAAA,IAAU,YAAY,OAAA,EAAS;AACxC,MAAA,IAAI;AAEF,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,MAAA,CAAO,IAAI,CAAA;AACpC,QAAA,eAAA,CAAgB,IAAA,CAAK;AAAA,UACnB,OAAO,MAAA,CAAO,IAAA;AAAA,UACd;AAAA,SACD,CAAA;AAAA,MACH,SAAS,KAAA,EAAO;AACd,QAAA,MAAA,CAAO,KAAA;AAAA,UACL,CAAA,4CAAA,EAA+C,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACvE;AACA,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,8BAAA,EAAiC,MAAA,CAAO,IAAI,CAAA,GAAA,EAAM,KAAK,CAAA;AAAA,SACzD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,sBAAA,EAAyB,eAAA,CACtB,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AACA,EAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,CAAA,sBAAA,EAAyB,eAAA,CACtB,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KACf;AAAA,EACF;AAGA,EAAA,MAAM,iBACJ,MAAA,CAAO,iBAAA;AAAA,IACL;AAAA,GACF,IAAKA,qDAAA;AAQP,EAAA,MAAM,wBAAwB,OAAO;AAAA,IACnC,IAAA;AAAA,IACA,KAAA;AAAA,IACA;AAAA,GACF,KAIM;AACJ,IAAA,MAAA,CAAO,IAAA;AAAA,MACL,cAAc,KAAA,CAAM,MAAM,qBAAqB,IAAA,CAAK,IAAI,mBAAmB,cAAc,CAAA;AAAA,KAC3F;AAEA,IAAA,IAAI,sBAAA,GAAyB,CAAA;AAK7B,IAAA,MAAM,YAAA,GAAe,IAAA,CAAK,IAAA,CAAK,KAAA,CAAM,SAAS,cAAc,CAAA;AAG5D,IAAA,KAAA,IACM,aAAa,CAAA,EACjB,UAAA,GAAa,KAAA,CAAM,MAAA,EACnB,cAAc,cAAA,EACd;AACA,MAAA,MAAM,WAAW,IAAA,CAAK,GAAA,CAAI,UAAA,GAAa,cAAA,EAAgB,MAAM,MAAM,CAAA;AACnE,MAAA,MAAM,UAAA,GAAa,KAAA,CAAM,KAAA,CAAM,UAAA,EAAY,QAAQ,CAAA;AACnD,MAAA,MAAM,WAAA,GAAc,IAAA,CAAK,KAAA,CAAM,UAAA,GAAa,cAAc,CAAA,GAAI,CAAA;AAE9D,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,iBAAA,EAAoB,WAAW,CAAA,CAAA,EAAI,YAAY,KAAK,UAAA,CAAW,MAAM,CAAA,kBAAA,EAAqB,IAAA,CAAK,IAAI,CAAA,CAAA;AAAA,OACrG;AAGA,MAAA,MAAM,YAAiC,EAAC;AAExC,MAAA,KAAA,IAAS,KAAA,GAAQ,CAAA,EAAG,KAAA,GAAQ,UAAA,CAAW,QAAQ,KAAA,EAAA,EAAS;AACtD,QAAA,MAAM,IAAA,GAAO,WAAW,KAAK,CAAA;AAC7B,QAAA,MAAM,cAAc,UAAA,GAAa,KAAA;AAEjC,QAAA,IAAI;AACF,UAAA,MAAM,CAAC,OAAA,EAAS,WAAW,CAAA,GAAI,MAAM,QAAQ,GAAA,CAAI;AAAA,YAC/C,kBAAA,CAAmB,kBAAA,CAAmB,IAAA,CAAK,EAAA,EAAK,KAAK,IAAK,CAAA;AAAA,YAC1D,kBAAA,CAAmB,sBAAA,CAAuB,IAAA,CAAK,EAAA,EAAK,KAAK,IAAK;AAAA,WAC/D,CAAA;AAED,UAAA,MAAM,eAAA,GAAkBC,iDAAA;AAAA,YACtB,WAAA,GAAc,CAAA;AAAA,YACd,KAAA,CAAM;AAAA,WACR;AAEA,UAAA,MAAA,CAAO,IAAA;AAAA,YACL,CAAA,0CAAA,EAA6C,IAAA,CAAK,IAAI,CAAA,YAAA,EAAe,KAAK,IAAI,CAAA,aAAA,EAAgB,eAAA,CAAgB,SAAS,CAAA,CAAA,EAAI,eAAA,CAAgB,KAAK,CAAA,EAAA,EAAK,gBAAgB,UAAU,CAAA,qBAAA;AAAA,WACjL;AAGA,UAAA,MAAM,WAAA,GAAc,MAAMC,6CAAA,CAAe,OAAO,CAAA;AAEhD,UAAA,MAAA,CAAO,KAAA;AAAA,YACL,CAAA,uBAAA,EAA0B,IAAA,CAAK,IAAI,CAAA,WAAA,EAAc,YAAY,MAAM,CAAA,CAAA;AAAA,WACrE;AAEA,UAAA,MAAM,OAAA,GAAU,IAAA,CAAK,SAAA,IAAa,IAAA,CAAK,GAAA;AAGvC,UAAA,IAAI,CAAC,WAAA,IAAe,WAAA,CAAY,IAAA,EAAK,CAAE,WAAW,CAAA,EAAG;AACnD,YAAA,MAAA,CAAO,IAAA;AAAA,cACL,CAAA,yCAAA,EAA4C,IAAA,CAAK,IAAI,CAAA,YAAA,EAAe,KAAK,IAAI,CAAA,YAAA;AAAA,aAC/E;AACA,YAAA;AAAA,UACF;AAEA,UAAA,MAAM,QAAA,GAA8B;AAAA,YAClC,QAAA,EAAU;AAAA,cACR,MAAA,EAAQC,kBAAA;AAAA,cACR,IAAI,CAAA,EAAG,IAAA,CAAK,EAAE,CAAA,CAAA,EAAI,KAAK,IAAI,CAAA,CAAA;AAAA,cAC3B,GAAA,EAAK,OAAA;AAAA,cACL,cAAc,kBAAA,CAAmB,YAAA;AAAA,cACjC,SAAS,kBAAA,CAAmB,OAAA;AAAA,cAC5B,MAAM,IAAA,CAAK;AAAA,aACb;AAAA,YACA,OAAA,EAAS,WAAA;AAAA,YACT;AAAA,WACF;AAEA,UAAA,MAAA,CAAO,KAAA;AAAA,YACL,CAAA,mDAAA,EAAsD,IAAA,CAAK,IAAI,CAAA,YAAA,EAAe,IAAA,CAAK,IAAI,CAAA,mBAAA,EAAsB,QAAA,CAAS,OAAA,CAAQ,MAAM,CAAA,cAAA,EAAiB,QAAA,CAAS,SAAS,GAAG,CAAA,CAAA;AAAA,WAC5K;AAEA,UAAA,SAAA,CAAU,KAAK,QAAQ,CAAA;AAAA,QACzB,SAAS,KAAA,EAAO;AACd,UAAA,MAAA,CAAO,IAAA;AAAA,YACL,sDAAsD,IAAA,CAAK,IAAI,eAAe,IAAA,CAAK,IAAI,aAAa,KAAK,CAAA,+BAAA;AAAA,WAC3G;AAAA,QAEF;AAAA,MACF;AAGA,MAAA,MAAM,mBAAmB,SAAS,CAAA;AAElC,MAAA,sBAAA,IAA0B,SAAA,CAAU,MAAA;AAEpC,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,MAAA,EAAS,WAAW,CAAA,CAAA,EAAI,YAAY,eAAe,SAAA,CAAU,MAAM,CAAA,2CAAA,EAA8C,IAAA,CAAK,IAAI,CAAA;AAAA,OAC5H;AAAA,IACF;AAEA,IAAA,OAAO,EAAE,sBAAA,EAAuB;AAAA,EAClC,CAAA;AAGA,EAAA,MAAM,gBAAA,GAAmB,OACvB,kBAAA,KACG;AACH,IAAA,MAAM,SAAA,GAAY,MAAM,kBAAA,CAAmB,QAAA,EAAS;AAEpD,IAAA,IAAI,SAAA,CAAU,WAAW,CAAA,EAAG;AAC1B,MAAA,MAAA,CAAO,KAAK,4CAA4C,CAAA;AACxD,MAAA;AAAA,IACF;AAGA,IAAA,IAAI,aAAA,GAAgB,SAAA;AAGpB,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,0FAAA,EAA6F,eAAA,CAC1F,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,aAAA,GAAgB,aAAA,CAAc,OAAO,CAAA,IAAA,KAAQ;AAC3C,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAAA,IACH;AAGA,IAAA,IAAI,eAAA,CAAgB,SAAS,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,sFAAA,EAAyF,eAAA,CACtF,GAAA,CAAI,CAAA,CAAA,KAAK,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAG,CAAA,CACvB,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,OACf;AAEA,MAAA,MAAM,aAAA,GAAgB,aAAA,CAAc,MAAA,CAAO,CAAA,IAAA,KAAQ;AACjD,QAAA,OAAO,eAAA,CAAgB,KAAK,CAAA,OAAA,KAAW,OAAA,CAAQ,MAAO,IAAA,CAAK,IAAA,CAAK,IAAK,CAAC,CAAA;AAAA,MACxE,CAAC,CAAA;AAED,MAAA,IAAI,aAAA,CAAc,SAAS,CAAA,EAAG;AAC5B,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,iBAAA,EAAoB,cAAc,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,SAC/D;AAAA,MACF;AACA,MAAA,aAAA,GAAgB,aAAA,CAAc,OAAO,CAAA,IAAA,KAAQ;AAC3C,QAAA,OAAO,CAAC,eAAA,CAAgB,IAAA;AAAA,UAAK,CAAA,OAAA,KAC3B,OAAA,CAAQ,KAAA,CAAO,IAAA,CAAK,KAAK,IAAK;AAAA,SAChC;AAAA,MACF,CAAC,CAAA;AAAA,IACH;AAEA,IAAA,IAAI,aAAA,CAAc,WAAW,CAAA,EAAG;AAC9B,MAAA,MAAA,CAAO,KAAK,wDAAwD,CAAA;AACpE,MAAA;AAAA,IACF;AAEA,IAAA,MAAA,CAAO,KAAA;AAAA,MACL,CAAA,iBAAA,EAAoB,cAAc,GAAA,CAAI,CAAA,CAAA,KAAK,EAAE,IAAI,CAAA,CAAE,IAAA,CAAK,IAAI,CAAC,CAAA;AAAA,KAC/D;AAEA,IAAA,MAAA,CAAO,IAAA,CAAK,CAAA,UAAA,EAAa,aAAA,CAAc,MAAM,CAAA,wBAAA,CAA0B,CAAA;AAGvE,IAAA,KAAA,MAAW,QAAQ,aAAA,EAAe;AAChC,MAAA,MAAA,CAAO,KAAK,CAAA,8BAAA,EAAiC,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,IAAA,CAAK,EAAE,CAAA,CAAA,CAAG,CAAA;AAGrE,MAAA,MAAM,KAAA,GAAQ,MAAM,kBAAA,CAAmB,YAAA,CAAa,KAAK,EAAG,CAAA;AAE5D,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,sDAAA,EAAyD,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,SAChF;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,EAAE,sBAAA,EAAuB,GAAI,MAAM,qBAAA,CAAsB;AAAA,QAC7D,IAAA;AAAA,QACA,KAAA;AAAA,QACA;AAAA,OACD,CAAA;AAED,MAAA,IAAI,2BAA2B,CAAA,EAAG;AAChC,QAAA,MAAA,CAAO,IAAA;AAAA,UACL,CAAA,6EAAA,EAAgF,IAAA,CAAK,IAAI,CAAA,EAAA,EAAK,KAAK,EAAE,CAAA,CAAA;AAAA,SACvG;AACA,QAAA;AAAA,MACF;AAEA,MAAA,MAAA,CAAO,IAAA;AAAA,QACL,CAAA,0BAAA,EAA6B,sBAAsB,CAAA,wEAAA,EAA2E,IAAA,CAAK,IAAI,CAAA;AAAA,OACzI;AAAA,IACF;AAAA,EACF,CAAA;AAEA,EAAA,OAAO,EAAE,gBAAA,EAAiB;AAC5B;;;;"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sweetoburrito/backstage-plugin-ai-assistant-backend-module-ingestor-azure-devops",
|
|
3
|
-
"version": "0.0.0-snapshot-
|
|
3
|
+
"version": "0.0.0-snapshot-20260112102353",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"description": "The ingestor-azure-devops backend module for the ai-assistant plugin.",
|
|
6
6
|
"main": "dist/index.cjs.js",
|