@firestartr/cli 2.7.0-snapshot-21 → 2.7.0-snapshot-22

package/build/index.js

@@ -330867,6 +330867,138 @@ function getOperatorEnvSnapshot() {
     return { ...snapshot };
 }
 
+;// CONCATENATED MODULE: ../operator/src/connection.ts
+
+
+async function connection_getConnection() {
+    try {
+        const kc = new client_node_dist.KubeConfig();
+        kc.loadFromDefault();
+        const opts = {};
+        await kc.applyToHTTPSOptions(opts);
+        return { kc, opts };
+    }
+    catch (err) {
+        operator_src_logger.error(`getConnection: ${err}`);
+        throw err;
+    }
+}
+
+;// CONCATENATED MODULE: ../operator/src/utils/semaphore.ts
+class Semaphore {
+    constructor(max) {
+        this._current = 0;
+        this._queue = [];
+        this._totalAcquisitions = 0;
+        this._timesSaturated = 0;
+        if (!Number.isFinite(max) || Number.isNaN(max) || max < 1) {
+            throw new Error(`Semaphore max must be a finite number >= 1: ${max}`);
+        }
+        this._max = Math.floor(max);
+    }
+    get max() {
+        return this._max;
+    }
+    get current() {
+        return this._current;
+    }
+    get waiting() {
+        return this._queue.length;
+    }
+    get totalAcquisitions() {
+        return this._totalAcquisitions;
+    }
+    get timesSaturated() {
+        return this._timesSaturated;
+    }
+    async acquire() {
+        if (this._current < this._max) {
+            this._current++;
+            this._totalAcquisitions++;
+            return;
+        }
+        this._timesSaturated++;
+        await new Promise((resolve) => {
+            this._queue.push(() => {
+                this._current++;
+                this._totalAcquisitions++;
+                resolve();
+            });
+        });
+    }
+    release() {
+        if (this._current <= 0) {
+            this._current = 0;
+            return;
+        }
+        this._current--;
+        const next = this._queue.shift();
+        if (next) {
+            next();
+        }
+    }
+}
+
+;// CONCATENATED MODULE: ../operator/src/utils/api-read-limiter.ts
+
+const DEFAULT_MAX_CONCURRENT_API_CALLS = 50;
+const MIN_CONCURRENT_API_CALLS_PER_SLOT = 4;
+function parseMaxSlots(value) {
+    if (value === undefined)
+        return 1;
+    const parsed = Number(value);
+    const floored = Math.floor(parsed);
+    if (!Number.isFinite(parsed) || Number.isNaN(parsed) || floored < 1)
+        return 1;
+    return floored;
+}
+function computeMaxConcurrentApiCalls(rawMaxConcurrent, rawMaxSlots) {
+    const maxSlots = parseMaxSlots(rawMaxSlots);
+    const minRequired = maxSlots * MIN_CONCURRENT_API_CALLS_PER_SLOT;
+    if (rawMaxConcurrent === undefined) {
+        return Math.max(DEFAULT_MAX_CONCURRENT_API_CALLS, minRequired);
+    }
+    const parsed = Number(rawMaxConcurrent);
+    const floored = Math.floor(parsed);
+    if (!Number.isFinite(parsed) || Number.isNaN(parsed) || floored < 1) {
+        return Math.max(DEFAULT_MAX_CONCURRENT_API_CALLS, minRequired);
+    }
+    return Math.max(floored, minRequired);
+}
+const apiReadSemaphore = new Semaphore(computeMaxConcurrentApiCalls(process.env.OPERATOR_MAX_CONCURRENT_API_CALLS, process.env.OPERATOR_NUMBER_OF_MAX_SLOTS));
+let _peakConcurrentApiCalls = 0;
+async function withApiReadSlot(read) {
+    await apiReadSemaphore.acquire();
+    const current = apiReadSemaphore.current;
+    if (current > _peakConcurrentApiCalls) {
+        _peakConcurrentApiCalls = current;
+    }
+    try {
+        return await read();
+    }
+    finally {
+        apiReadSemaphore.release();
+    }
+}
+function getPeakConcurrentApiCalls() {
+    return _peakConcurrentApiCalls;
+}
+function getCurrentConcurrentApiCalls() {
+    return apiReadSemaphore.current;
+}
+function getPendingApiCalls() {
+    return apiReadSemaphore.waiting;
+}
+function getMaxConcurrentApiCalls() {
+    return apiReadSemaphore.max;
+}
+function getTotalApiAcquisitions() {
+    return apiReadSemaphore.totalAcquisitions;
+}
+function getTimesSemaphoreSaturated() {
+    return apiReadSemaphore.timesSaturated;
+}
+
 ;// CONCATENATED MODULE: ../operator/src/ctl.ts
 
 
@@ -330876,6 +331008,8 @@ function getOperatorEnvSnapshot() {
 
 
 
+
+
 const MAX_CHARS_OUPUT_PLAN = 20000;
 function getOperatorProfile() {
     github["default"].createProfile('operator', {
@@ -330888,29 +331022,33 @@ async function getItem(kind, namespace, item) {
     const itemPath = `${namespace}/${kind}/${item.metadata.name}`;
     return await ctl_getItemByItemPath(itemPath);
 }
+async function fetchItemByItemPathResponse(itemPath, apiGroup = catalog_common/* default.types.controller.FirestartrApiGroup */.Z.types.controller.FirestartrApiGroup, apiVersion = 'v1') {
+    operator_src_logger.debug(`The ctl is getting the item at '${itemPath}'.`);
+    const { kc, opts } = await connection_getConnection();
+    opts.headers['Content-Type'] = 'application/json';
+    opts.headers['Accept'] = 'application/json';
+    const url = apiGroup === catalog_common/* default.types.controller.KubernetesApiGroup */.Z.types.controller.KubernetesApiGroup
+        ? `${kc.getCurrentCluster().server}/api/v1/namespaces/${itemPath}`
+        : `${kc.getCurrentCluster().server}/apis/${apiGroup}/${apiVersion}/namespaces/${itemPath}`;
+    operator_src_logger.silly(`Sending request ${url}`);
+    return await fetch(url, { method: 'get', headers: opts.headers });
+}
 async function ctl_getItemByItemPath(itemPath, apiGroup = catalog_common/* default.types.controller.FirestartrApiGroup */.Z.types.controller.FirestartrApiGroup, apiVersion = 'v1') {
-    try {
-        operator_src_logger.debug(`The ctl is getting the item at '${itemPath}'.`);
-        const { kc, opts } = await ctl_getConnection();
-        opts.headers['Content-Type'] = 'application/json';
-        opts.headers['Accept'] = 'application/json';
-        const url = apiGroup === catalog_common/* default.types.controller.KubernetesApiGroup */.Z.types.controller.KubernetesApiGroup
-            ? `${kc.getCurrentCluster().server}/api/v1/namespaces/${itemPath}`
-            : `${kc.getCurrentCluster().server}/apis/${apiGroup}/${apiVersion}/namespaces/${itemPath}`;
-        operator_src_logger.silly(`Sending request ${url}`);
-        const r = await fetch(url, { method: 'get', headers: opts.headers });
-        if (!r.ok) {
-            const err = new Error(`Error on getItemByItemPath: ${itemPath}: ${r.statusText}`);
-            operator_src_logger.error(`Error on getItemByItemPath: ${itemPath}: ${r.statusText}`);
-            throw err;
+    return await withApiReadSlot(async () => {
+        try {
+            const r = await fetchItemByItemPathResponse(itemPath, apiGroup, apiVersion);
+            if (!r.ok) {
+                const err = new Error(`Error on getItemByItemPath: ${itemPath}: ${r.statusText}`);
+                operator_src_logger.error(`Error on getItemByItemPath: ${itemPath}: ${r.statusText}`);
+                throw err;
+            }
+            return await r.json();
         }
-        const jsonResponse = await r.json();
-        return jsonResponse;
-    }
-    catch (e) {
-        operator_src_logger.error(`Error on getItemByItemPath: ${e}`);
-        throw e;
-    }
+        catch (e) {
+            operator_src_logger.error(`Error on getItemByItemPath: ${e}`);
+            throw e;
+        }
+    });
 }
 /**
  * Like getItemByItemPath but returns null when the resource is not found
@@ -330919,34 +331057,28 @@ async function ctl_getItemByItemPath(itemPath, apiGroup = catalog_common/* defau
  * whether a parent CR has been provisioned before its children arrive).
  */
 async function getItemByItemPathOrNull(itemPath, apiGroup = catalog_common/* default.types.controller.FirestartrApiGroup */.Z.types.controller.FirestartrApiGroup, apiVersion = 'v1') {
-    try {
-        operator_src_logger.debug(`The ctl is getting the item at '${itemPath}'.`);
-        const { kc, opts } = await ctl_getConnection();
-        opts.headers['Content-Type'] = 'application/json';
-        opts.headers['Accept'] = 'application/json';
-        const url = apiGroup === catalog_common/* default.types.controller.KubernetesApiGroup */.Z.types.controller.KubernetesApiGroup
-            ? `${kc.getCurrentCluster().server}/api/v1/namespaces/${itemPath}`
-            : `${kc.getCurrentCluster().server}/apis/${apiGroup}/${apiVersion}/namespaces/${itemPath}`;
-        operator_src_logger.silly(`Sending request ${url}`);
-        const r = await fetch(url, { method: 'get', headers: opts.headers });
-        if (r.status === 404 || r.status === 409) {
-            operator_src_logger.debug(`Item at '${itemPath}' returned ${r.status} (expected), returning null.`);
-            return null;
+    return await withApiReadSlot(async () => {
+        try {
+            const r = await fetchItemByItemPathResponse(itemPath, apiGroup, apiVersion);
+            if (r.status === 404 || r.status === 409) {
+                operator_src_logger.debug(`Item at '${itemPath}' returned ${r.status} (expected), returning null.`);
+                return null;
+            }
+            if (!r.ok) {
+                const err = new Error(`Error on getItemByItemPathOrNull: ${itemPath}: ${r.statusText}`);
+                operator_src_logger.error(`Error on getItemByItemPathOrNull: ${itemPath}: ${r.statusText}`);
+                throw err;
+            }
+            return await r.json();
         }
-        if (!r.ok) {
-            const err = new Error(`Error on getItemByItemPath: ${itemPath}: ${r.statusText}`);
-            operator_src_logger.error(`Error on getItemByItemPath: ${itemPath}: ${r.statusText}`);
-            throw err;
+        catch (e) {
+            operator_src_logger.error(`Error on getItemByItemPathOrNull: ${e}`);
+            throw e;
         }
-        return await r.json();
-    }
-    catch (e) {
-        operator_src_logger.error(`Error on getItemByItemPath: ${e}`);
-        throw e;
-    }
+    });
 }
 async function writeManifest(kind, namespace, item, apiSlug) {
-    const { kc, opts } = await ctl_getConnection();
+    const { kc, opts } = await connection_getConnection();
     opts.headers['Content-Type'] = 'application/json';
     opts.headers['Accept'] = 'application/json';
     const r = await fetch(`${kc.getCurrentCluster().server}/${apiSlug}`, {
@@ -331028,19 +331160,8 @@ function helperIsValidJSON(part) {
         return false;
     }
 }
-async function ctl_getConnection() {
-    try {
-        const kc = new client_node_dist.KubeConfig();
-        kc.loadFromDefault();
-        const opts = {};
-        await kc.applyToHTTPSOptions(opts);
-        return { kc, opts };
-    }
-    catch (err) {
-        operator_src_logger.error(`getConnection: ${err}`);
-        throw err;
-    }
-}
+
+
 function buildTFResultStatusUpdate(namespace, tfResultName, resourceVersion, exitCode, retryCount) {
     const status = {
         exitCode: exitCode,
@@ -331074,7 +331195,7 @@ async function tryWriteTFResultStatus(namespace, tfResultName, resourceVersion,
     }
 }
 async function upsertResult(namespace, item, result, exitCode = 0, existingRetryCount) {
-    const { kc, opts } = await ctl_getConnection();
+    const { kc, opts } = await connection_getConnection();
     kc.makeApiClient(client_node_dist.CoreV1Api);
     const exists = await getTFResult(namespace, item);
     const currentRetryCount = existingRetryCount !== undefined
@@ -331129,7 +331250,7 @@ async function upsertResult(namespace, item, result, exitCode = 0, existingRetry
     }
 }
 async function deleteSecret(secretName, namespace) {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CoreV1Api);
     try {
         await k8sApi.deleteNamespacedSecret(secretName, namespace);
@@ -331145,7 +331266,7 @@ async function deleteSecret(secretName, namespace) {
     }
 }
 async function upsertSecret(namespace, secret) {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CoreV1Api);
     secret.metadata.name = secret.metadata.name.toLowerCase();
     try {
@@ -331162,7 +331283,7 @@ async function upsertSecret(namespace, secret) {
     }
 }
 async function getSecret(namespace, secretName) {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CoreV1Api);
     try {
         const secret = await k8sApi.readNamespacedSecret(secretName, namespace);
@@ -331179,7 +331300,7 @@ async function getSecret(namespace, secretName) {
     }
 }
 async function getTFResult(namespace, item) {
-    const { kc, opts } = await ctl_getConnection();
+    const { kc, opts } = await connection_getConnection();
     opts.headers['Content-Type'] = 'application/json';
     opts.headers['Accept'] = 'application/json';
     const r = await fetch(`${kc.getCurrentCluster().server}/apis/firestartr.dev/v1/namespaces/${namespace}/tfresults/${item.kind}-${item.metadata.name}`.toLowerCase(), {
@@ -331209,7 +331330,7 @@ async function checkIfRenamed(namespace, item) {
     try {
         // Check that the old item still exists
         const plural = getPluralFromKind(item.kind);
-        const { kc, opts } = await ctl_getConnection();
+        const { kc, opts } = await getCtlConnection();
         const url = `${kc.getCurrentCluster().server}/apis/firestartr.dev/v1/namespaces/${namespace}/${plural}/${item.metadata.name}`;
         const r = await fetch(url, {
             method: 'GET',
@@ -331230,7 +331351,7 @@ async function checkIfRenamed(namespace, item) {
     }
 }
 async function upsertFinalizer(kind, namespace, item, finalizer) {
-    const { kc, opts } = await ctl_getConnection();
+    const { kc, opts } = await connection_getConnection();
     const url = `${kc.getCurrentCluster().server}/apis/firestartr.dev/v1/namespaces/${namespace}/${kind}/${item.metadata.name}`;
     for (let attempt = 0; attempt < 5; attempt++) {
         // Fresh GET to get current finalizers and resourceVersion
@@ -331286,7 +331407,7 @@ async function upsertFinalizer(kind, namespace, item, finalizer) {
  * @returns {void}
  */
 async function unsetFinalizer(kind, namespace, item, finalizer) {
-    const { kc, opts } = await ctl_getConnection();
+    const { kc, opts } = await connection_getConnection();
     let itemObj;
     if (typeof item === 'string') {
         itemObj = await ctl_getItemByItemPath([namespace, kind, item].join('/'));
@@ -331457,7 +331578,7 @@ async function addCommitStatusToPrMergeCommit(prAnnotationValue, state, targetUR
     await getOperatorProfile().repo.addCommitStatus(state, branchSha, repo, owner, targetURL, description, context);
 }
 async function createDryRun(manifest, namespace) {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CustomObjectsApi);
     const plural = definitions_getPluralFromKind(manifest.kind);
     if (!plural)
@@ -331465,7 +331586,7 @@ async function createDryRun(manifest, namespace) {
     await k8sApi.createNamespacedCustomObject('firestartr.dev', 'v1', namespace, plural, manifest, undefined, 'All');
 }
 async function updateDryRun(manifest, namespace) {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CustomObjectsApi);
     const plural = definitions_getPluralFromKind(manifest.kind);
     if (!plural)
@@ -331489,7 +331610,7 @@ async function updateDryRun(manifest, namespace) {
  */
 async function observe(plural, namespace, onAdd, onChange, onDelete, _onRename) {
     const store = new Store(plural);
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     try {
         const k8sApi = kc.makeApiClient(client_node_dist.CustomObjectsApi);
         const apiGroup = 'firestartr.dev';
@@ -331870,10 +331991,10 @@ async function getSyncSpecs(itemPath, itemCR) {
     return {
         item,
         syncable: helperIsSyncable(item),
-        period: item.metadata.annotations['firestartr.dev/sync-period'] ||
+        period: item.metadata.annotations?.['firestartr.dev/sync-period'] ||
             DEFAULT_REVISION_TIME,
-        schedule: item.metadata.annotations['firestartr.dev/sync-schedule'] || false,
-        scheduleTZ: item.metadata.annotations['firestartr.dev/sync-schedule-timezone'] ||
+        schedule: item.metadata.annotations?.['firestartr.dev/sync-schedule'] || false,
+        scheduleTZ: item.metadata.annotations?.['firestartr.dev/sync-schedule-timezone'] ||
             'Europe/Madrid',
     };
 }
@@ -331924,7 +332045,7 @@ function assessSyncCalculationMachinery(item) {
     if (!helperIsSyncable(item)) {
         return processNotSyncable;
     }
-    else if (item.metadata.annotations['firestartr.dev/sync-schedule'] ??
+    else if (item.metadata.annotations?.['firestartr.dev/sync-schedule'] ??
         false) {
         return processScheduledSync;
     }
@@ -331953,7 +332074,8 @@ async function processNotSyncable(item, reason, status, message) {
     };
 }
 async function processPeriodSync(item, reason, status, message) {
-    const period = item.metadata.annotations['firestartr.dev/sync-period'];
+    const period = item.metadata.annotations?.['firestartr.dev/sync-period'] ||
+        DEFAULT_REVISION_TIME;
     const periodMS = helperCalculateRevisionTime(period);
     return {
         syncMode: 'Period',
@@ -331983,7 +332105,7 @@ function helperCalculateRevisionTime(period) {
     return Number(scalar) * multiplier * 1000;
 }
 async function processScheduledSync(item, reason, status, message) {
-    const nextPeriod = catalog_common/* default.cron.getCronNextInterval */.Z.cron.getCronNextInterval(item.metadata.annotations['firestartr.dev/sync-schedule'], item.metadata.annotations['firestartr.dev/sync-schedule-timezone'] ||
+    const nextPeriod = catalog_common/* default.cron.getCronNextInterval */.Z.cron.getCronNextInterval(item.metadata.annotations?.['firestartr.dev/sync-schedule'], item.metadata.annotations?.['firestartr.dev/sync-schedule-timezone'] ||
         undefined);
     const nextPeriodDate = new Date(nextPeriod);
     return {
@@ -332888,7 +333010,7 @@ async function* traverseCRCollection(pluralKind, namespace, options = {}, apiGro
     const { limit = 300 } = options; // between 250-300
     try {
         let continueToken = undefined;
-        const { kc, opts } = await ctl_getConnection();
+        const { kc, opts } = await connection_getConnection();
         const k8sApi = kc.makeApiClient(client_node_dist.CustomObjectsApi);
         const apiGroup = 'firestartr.dev';
         const apiVersion = 'v1';
@@ -333462,6 +333584,165 @@ function formatElapsedTimeWithDate(upsertTime) {
     return `${parts.slice(0, 2).join(', ')} ago`;
 }
 
+;// CONCATENATED MODULE: ../operator/src/diagnosticErrors.ts
+/**
+ * Global error counter for the diagnostic file.
+ * Increment-only; never reset during the operator lifetime.
+ */
+let _errorCount = 0;
+function incrementDiagnosticErrorCount() {
+    _errorCount++;
+}
+function getDiagnosticErrorCount() {
+    return _errorCount;
+}
+
+;// CONCATENATED MODULE: ../operator/src/processItem.diagnostic.ts
+
+
+
+
+
+const DIAGNOSTIC_FILE = '/tmp/diagnostic';
+const DIAGNOSTIC_INTERVAL_MS = 10 * 1000;
+let _diagnosticTimer;
+function clearTimer() {
+    if (_diagnosticTimer !== undefined) {
+        clearInterval(_diagnosticTimer);
+        _diagnosticTimer = undefined;
+    }
+}
+function loopDiagnosticFile(queue, maxSlots) {
+    if (process.env.DISABLE_DIAGNOSTIC_FILE) {
+        operator_src_logger.info('Diagnostic file disabled by DISABLE_DIAGNOSTIC_FILE');
+        return;
+    }
+    let running = false;
+    _diagnosticTimer = setInterval(() => {
+        if (running)
+            return;
+        running = true;
+        writeDiagnosticSnapshot(queue, maxSlots)
+            .then(() => {
+            running = false;
+        })
+            .catch((err) => {
+            running = false;
+            operator_src_logger.error(`Error writing diagnostic file: ${err}`);
+        });
+    }, DIAGNOSTIC_INTERVAL_MS);
+    // Clear the interval on termination signals so the process can exit cleanly
+    const onSignal = () => clearTimer();
+    process.on('SIGTERM', onSignal);
+    process.on('SIGINT', onSignal);
+}
+/** @internal for testing */
+function getDiagnosticTimer() {
+    return _diagnosticTimer;
+}
+async function writeDiagnosticSnapshot(queue, maxSlots) {
+    const now = new Date().toISOString();
+    // Queue counts
+    let pending = 0;
+    let processing = 0;
+    let finished = 0;
+    let blocked = 0;
+    let deadLetter = 0;
+    // Per-slot items
+    const slotsMap = {};
+    for (const wi of queue) {
+        switch (wi.workStatus) {
+            case WorkStatus.PENDING:
+                pending++;
+                break;
+            case WorkStatus.PROCESSING:
+                processing++;
+                break;
+            case WorkStatus.FINISHED:
+                finished++;
+                break;
+        }
+        if (wi.isBlocked) {
+            blocked++;
+        }
+        if (wi.isDeadLetter) {
+            deadLetter++;
+        }
+        if (wi.workStatus === WorkStatus.PROCESSING && wi.slotId !== undefined) {
+            slotsMap[`slot-${wi.slotId}`] =
+                `${wi.item.kind}/${wi.item.metadata.name}`;
+        }
+    }
+    // Fill empty slots with null
+    for (let i = 0; i < maxSlots; i++) {
+        if (!slotsMap[`slot-${i}`]) {
+            slotsMap[`slot-${i}`] = 'null';
+        }
+    }
+    // Semaphore stats
+    const semaphoreCurrent = getCurrentConcurrentApiCalls();
+    const semaphoreMax = getMaxConcurrentApiCalls();
+    const semaphorePeak = getPeakConcurrentApiCalls();
+    const semaphorePending = getPendingApiCalls();
+    const semaphoreTotalAcquisitions = getTotalApiAcquisitions();
+    const semaphoreTimesSaturated = getTimesSemaphoreSaturated();
+    // Environment
+    const envMaxSlots = process.env.OPERATOR_NUMBER_OF_MAX_SLOTS || '1';
+    const envMaxConcurrentApi = process.env.OPERATOR_MAX_CONCURRENT_API_CALLS || '25';
+    const envKindList = process.env.OPERATOR_KIND_LIST || '';
+    const envTfmMirrorDisable = process.env.TFM_MIRROR_DISABLE || '0';
+    const envTfmMirrorList = process.env.TFM_MIRROR_LIST || '';
+    const envTfmMirrorRefreshInterval = process.env.TFM_MIRROR_REFRESH_INTERVAL || '900';
+    const envBackendProvider = process.env.BACKEND_PROVIDER_NAME || 'kubernetes-provider';
+    const envGithubAppId = process.env.GITHUB_APP_ID || '';
+    // Build YAML output
+    let output = '';
+    output += `timestamp: "${now}"\n`;
+    output += 'env:\n';
+    output += `  OPERATOR_NUMBER_OF_MAX_SLOTS: ${envMaxSlots}\n`;
+    output += `  OPERATOR_MAX_CONCURRENT_API_CALLS: ${envMaxConcurrentApi}\n`;
+    output += `  OPERATOR_KIND_LIST: ${envKindList}\n`;
+    output += `  TFM_MIRROR_DISABLE: ${envTfmMirrorDisable}\n`;
+    output += `  TFM_MIRROR_LIST: ${envTfmMirrorList}\n`;
+    output += `  TFM_MIRROR_REFRESH_INTERVAL: ${envTfmMirrorRefreshInterval}\n`;
+    output += `  BACKEND_PROVIDER_NAME: ${envBackendProvider}\n`;
+    output += `  GITHUB_APP_ID: ${envGithubAppId}\n`;
+    output += 'queue:\n';
+    output += `  total: ${queue.length}\n`;
+    output += `  pending: ${pending}\n`;
+    output += `  processing: ${processing}\n`;
+    output += `  finished: ${finished}\n`;
+    output += `  blocked: ${blocked}\n`;
+    output += `  dead_letter: ${deadLetter}\n`;
+    output += 'slots:\n';
+    output += `  max: ${maxSlots}\n`;
+    output += `  active: ${processing}\n`;
+    output += '  items_per_slot:\n';
+    for (let i = 0; i < maxSlots; i++) {
+        const key = `slot-${i}`;
+        const val = slotsMap[key];
+        output += `    ${key}: ${val === 'null' ? 'null' : `"${val}"`}\n`;
+    }
+    output += 'semaphore:\n';
+    output += `  max: ${semaphoreMax}\n`;
+    output += `  in_use: ${semaphoreCurrent}\n`;
+    output += `  available: ${semaphoreMax - semaphoreCurrent}\n`;
+    output += `  pending: ${semaphorePending}\n`;
+    output += `  peak: ${semaphorePeak}\n`;
+    output += `  total_acquisitions: ${semaphoreTotalAcquisitions}\n`;
+    output += `  times_saturated: ${semaphoreTimesSaturated}\n`;
+    output += 'errors:\n';
+    output += `  total: ${getDiagnosticErrorCount()}\n`;
+    return new Promise((resolve, reject) => {
+        external_fs_.writeFile(DIAGNOSTIC_FILE, output, (err) => {
+            if (err)
+                reject(`Writing ${DIAGNOSTIC_FILE}: ${err}`);
+            else
+                resolve();
+        });
+    });
+}
+
 ;// CONCATENATED MODULE: ../operator/src/signals.ts
 
 function initSignalsHandler(Mapper) {
@@ -333484,6 +333765,7 @@ function initSignalsHandler(Mapper) {
 
 
 
+
 /*
  * Dead-Letter Handler
  *
@@ -333495,6 +333777,7 @@ async function deadLetterHandler(workItem) {
     const itemPath = workItem.handler.itemPath();
     workItem.workStatus = WorkStatus.FINISHED;
     workItem.isDeadLetter = true;
+    incrementDiagnosticErrorCount();
     // we try to put the CR in a correct state
     try {
         await updateTransition(itemPath, operation, 'ERROR', 'True', 'Uncontrolled error (DLH)');
@@ -333639,7 +333922,7 @@ class CRStateMetrics {
         this.onUpdate = false;
     }
     async __prepareConnection() {
-        const { kc, opts } = await ctl_getConnection();
+        const { kc, opts } = await connection_getConnection();
         const k8sApi = kc.makeApiClient(client_node_dist.CustomObjectsApi);
         this.fListCRs = async () => {
             try {
@@ -333783,6 +334066,7 @@ ProcessorGlobalMetrics._instance = null;
 
 
 
+
 let processMetricsSlotProvider = null;
 /* harmony default export */ async function metricsServer(kindList, namespace) {
     const portFromEnv = process.env.METRICS_PORT;
@@ -333801,6 +334085,7 @@ async function startMetrics(exporter, attributes, kindList, namespace) {
     });
     const meter = meterProvider.getMeter('firestartr');
     void startQueueMetrics(meter, attributes);
+    void startApiCallMetrics(meter, attributes);
     void startMemoryMetrics(meter, attributes);
     void startCRStates(meter, kindList, namespace);
     processMetricsSlotProvider = (slotId) => {
@@ -333862,6 +334147,14 @@ async function startQueueMetrics(meter, attributes) {
         nDeferred = queueMetrics.nItemsDeferred || 0;
     }, 1000);
 }
+async function startApiCallMetrics(meter, attributes) {
+    const concurrentGauge = meter.createObservableGauge('firestartr_api_calls_concurrent', { description: 'Current number of in-flight Kubernetes API read calls' });
+    const awaitingGauge = meter.createObservableGauge('firestartr_api_calls_awaiting', {
+        description: 'Current number of Kubernetes API read calls waiting for a slot',
+    });
+    concurrentGauge.addCallback((observer) => observer.observe(getCurrentConcurrentApiCalls(), attributes));
+    awaitingGauge.addCallback((observer) => observer.observe(getPendingApiCalls(), attributes));
+}
 async function startMemoryMetrics(meter, attributes) {
     const usedMemoryCounter = meter.createObservableGauge('firestartr_used_memory', { description: 'Used memory in bytes' });
     let heapUsage = process.memoryUsage().heapUsed;
@@ -334010,7 +334303,15 @@ class ProcessItemSlot {
     async __run(workItem) {
         if (!workItem.getItem || !workItem.process || !workItem.operation)
             return;
+        const raceDelayMs = process.env.RACE_DELAY_MS
+            ? Number.parseInt(process.env.RACE_DELAY_MS, 10)
+            : 0;
+        if (raceDelayMs > 0) {
+            const jitter = Math.floor(Math.random() * raceDelayMs);
+            await new Promise((r) => setTimeout(r, jitter));
+        }
         const item = await workItem.getItem();
+        workItem.item = item;
         // we check if the workItem needs blocking
         // if it does need it we return because we cannot
         // process this item
@@ -334231,6 +334532,7 @@ function sortQueue(queue) {
 
 
 
+
 const queue = [];
 let _deferredBookkeeper = null;
 let _activeParentKeys = new Set();
@@ -334340,6 +334642,7 @@ async function processItem(workItem) {
  */
 async function processItem_loop() {
     loopWorkItemDebug(queue);
+    loopDiagnosticFile(queue, MAX_SLOTS);
     const deferredBookkeeper = new DeferredSchedulingBookkeeper();
     _deferredBookkeeper = deferredBookkeeper;
     const processItemSlotsManager = initProcessItemsSlots(MAX_SLOTS, true, (w) => deferredBookkeeper.incrementBlockedAttempt(w));
@@ -337804,7 +338107,7 @@ var external_os_default = /*#__PURE__*/__nccwpck_require__.n(external_os_);
 
 
 async function getCurrentPod(namespace = 'default') {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CoreV1Api);
     const podName = (0,external_os_.hostname)();
     const pod = await k8sApi.readNamespacedPod(podName, namespace);
@@ -337826,7 +338129,7 @@ class LeaseAcquisitionError extends Error {
  * automatically after 30 seconds.
  */
 async function tryAcquireOrRenewLease(namespace, leaseDurationSeconds) {
-    const { kc } = await ctl_getConnection();
+    const { kc } = await connection_getConnection();
     const k8sApi = kc.makeApiClient(client_node_dist.CoordinationV1Api);
     const name = 'firestartr-lease';
     const currentPod = await getCurrentPod(namespace);
@@ -338405,7 +338708,7 @@ class Ctx {
 
 
 async function getTFWorkspaceRefs(_claimKind, claimName, namespace) {
-    const { kc, opts } = await ctl_getConnection();
+    const { kc, opts } = await connection_getConnection();
     await kc.makeApiClient(client_node_dist.CustomObjectsApi);
     const normalizedClaimName = catalog_common/* default.generic.normalizeLabel */.Z.generic.normalizeLabel(claimName);
     console.log('💡', normalizedClaimName);
@@ -338659,8 +338962,8 @@ const MODULES = {
     },
     FirestartrGithubOrganizationSettings: {
         module: 'git::https://github.com/prefapp/tfm.git//modules/github-org-settings',
-        // feat/org-settings-module at 2026-06-16
-        ref: '0e3f3f897ffff35a52285d0c681c0d81026d6d8a',
+        // org settings module merged to main (prefapp/tfm#1253)
+        ref: 'cf2e2ae487b6cc3f7b208f46a92f5b5f97f8fef2',
     },
 };
 function getFirestartrDefaultTFM(kind) {
@@ -339777,11 +340080,14 @@ async function provisionManagedFiles(ghFeature) {
             // the repo if available so the seed commit preserves the current state.
             newlyProvisionedAddresses.push(address);
             try {
-                const content = await ghFeature.getFileContentFromProvider(cr.spec.org, `/${cr.spec.org}/${repoRef.getDepName()}/${branch}/${file.path}`);
+                const content = await ghFeature.getFileContentFromProvider(cr.spec.org, repoRef.getDepName(), branch, file.path);
                 file.content = Buffer.from(content, 'binary').toString('base64');
             }
             catch (e) {
-                gh_provisioner_src_logger.error(`File ${file.path} not found in repo ${repoInfo.ref?.name} on branch ${branch}. File content not updated`);
+                if (e?.status !== 404) {
+                    throw e;
+                }
+                gh_provisioner_src_logger.info(`File ${file.path} not found in repo ${repoInfo.ref?.name} on branch ${branch}. Using declared content`);
             }
             createFileProvision(ghFeature, file);
         }
@@ -339965,6 +340271,7 @@ async function endDebug(entity) {
 
 
 const tp_bridge_TF_PROJECTS_PATH = '/tmp/gh-workspaces';
+const KUBERNETES_LABEL_VALUE_MAX_LENGTH = 63;
 async function runOnTerraform(entity, command, customArgs, opts = {}) {
     if (!entity || typeof entity !== 'object')
         throw new Error('[BUG] entity is invalid in runOnTerraform');
@@ -340054,11 +340361,27 @@ function calculateTFStatePath(entity, backend) {
         tfStatePath = `grss/${entity.cr.spec.firestartr.tfStateKey}`.substring(0, 63);
     }
     else {
-        tfStatePath = `${entity.cr.kind.toLowerCase()}/${entity.cr.spec.firestartr.tfStateKey}`;
+        const kind = entity.cr.kind.toLowerCase();
+        const tfStateKey = entity.cr.spec.firestartr.tfStateKey;
+        tfStatePath = `${kind}/${tfStateKey}`;
+        if ('kubernetes' in backend) {
+            tfStatePath = fitKubernetesBackendLabel(tfStatePath, kind, tfStateKey);
+        }
     }
     gh_provisioner_src_logger.debug(`[gh-provisioner] tfStateKey for ${entity.k8sId} = '${tfStatePath}'`);
     return tfStatePath;
 }
+function fitKubernetesBackendLabel(tfStatePath, kind, tfStateKey) {
+    if (tfStatePath.split('/').join('-').length <= KUBERNETES_LABEL_VALUE_MAX_LENGTH) {
+        return tfStatePath;
+    }
+    const tfStateKeyLabelLength = tfStateKey.split('/').join('-').length;
+    const maxKindLength = KUBERNETES_LABEL_VALUE_MAX_LENGTH - tfStateKeyLabelLength - 1;
+    if (maxKindLength < 1) {
+        return tfStateKey.substring(0, KUBERNETES_LABEL_VALUE_MAX_LENGTH);
+    }
+    return `${kind.substring(0, maxKindLength)}/${tfStateKey}`;
+}
 
 ;// CONCATENATED MODULE: ../gh_provisioner/src/entities/ghfeature/index.ts
 
@@ -340219,15 +340542,9 @@ class EntityGHFeature extends base_Entity {
             throw new Error(`[gh-provisioner] ${this.k8sId} on untracking files: ${err}`);
         }
     }
-    async getFileContentFromProvider(org, url) {
-        const v = await this.runWithGithubProvider(async () => {
-            const octokit = await github["default"].getOctokitForOrg(org);
-            const response = await octokit.request(`GET ${url}`, {
-                baseUrl: 'https://raw.githubusercontent.com',
-            });
-            return response.data;
-        });
-        return v;
+    async getFileContentFromProvider(org, repo, branch, path) {
+        const content = await this.runWithGithubProvider(() => github["default"].repo.getContent(path, repo, org, branch));
+        return content;
     }
 }
 function parseStateList(output) {
@@ -342173,6 +342490,7 @@ function isObserveModeActive() {
 // (warmup and legacy provisioner code replaced by TFM mirror system)
 function runOperator(opts) {
     const { ignoreLease, dummyExec, namespace, kindList, importMode, importModeSkipPlan, observeMode, withMetrics, } = opts;
+    const implementedKinds = filterImplementedKinds(kindList);
     if (observeMode)
         observeModeActive = true;
     if (withMetrics)
@@ -342206,7 +342524,7 @@ function runOperator(opts) {
             await leasePromise;
         };
     if (withMetricsMode) {
-        void metricsServer(kindList, namespace);
+        void metricsServer(implementedKinds, namespace);
     }
     void (async () => {
         await run(namespace, () => {
@@ -342214,7 +342532,7 @@ function runOperator(opts) {
                 void observeKind('githubgroups', namespace, processItem, dummy);
             }
             else {
-                for (const kind of kindList) {
+                for (const kind of implementedKinds) {
                     observeKind(kind, namespace, processItem, getProvisionImplementation(kind))
                         .then((res) => {
                         console.log('exit kind', kind);
@@ -342236,30 +342554,31 @@ function runOperator(opts) {
         });
     })();
 }
+const provisionImplementations = {
+    terraformworkspaces: processOperation,
+    terraformworkspaceplans: processOperationPlan,
+    githubgroups: process_operation_processOperation,
+    githubrepositories: process_operation_processOperation,
+    githubrepositorysecretssections: process_operation_processOperation,
+    githubrepositoryfeatures: process_operation_processOperation,
+    githubmemberships: process_operation_processOperation,
+    githuborgwebhooks: process_operation_processOperation,
+    githuborganizationsettings: process_operation_processOperation,
+    fsdummiesa: processFirestartrDummies,
+    fsdummiesb: processFirestartrDummies,
+    fsdummiesc: processFirestartrDummies,
+};
+function filterImplementedKinds(kindList) {
+    return kindList.filter((kind) => {
+        if (Object.prototype.hasOwnProperty.call(provisionImplementations, kind)) {
+            return true;
+        }
+        operator_src_logger.warn(`Ignoring configured kind '${kind}': no implementation found`);
+        return false;
+    });
+}
 function getProvisionImplementation(plural) {
-    let implementation = undefined;
-    switch (plural) {
-        case 'terraformworkspaces':
-            implementation = processOperation;
-            break;
-        case 'terraformworkspaceplans':
-            implementation = processOperationPlan;
-            break;
-        case 'githubgroups':
-        case 'githubrepositories':
-        case 'githubrepositorysecretssections':
-        case 'githubrepositoryfeatures':
-        case 'githubmemberships':
-        case 'githuborgwebhooks':
-        case 'githuborganizationsettings':
-            implementation = process_operation_processOperation;
-            break;
-        case 'fsdummiesa':
-        case 'fsdummiesb':
-        case 'fsdummiesc':
-            implementation = processFirestartrDummies;
-            break;
-    }
+    const implementation = provisionImplementations[plural];
     if (!implementation)
         throw new Error(`No implementation found for ${plural}`);
     operator_src_logger.info(`Retrieved the provision implementation for the kind '${plural}'`);
@@ -521050,7 +521369,7 @@ const crsStatusSubcommand = {
 };
 
 ;// CONCATENATED MODULE: ./package.json
-const package_namespaceObject = JSON.parse('{"i8":"2.7.0-snapshot-21"}');
+const package_namespaceObject = JSON.parse('{"i8":"2.7.0-snapshot-22"}');
 ;// CONCATENATED MODULE: ../../package.json
 const package_namespaceObject_1 = {"i8":"2.7.0"};
 ;// CONCATENATED MODULE: ./src/subcommands/index.ts

package/build/packages/gh_provisioner/src/entities/ghfeature/index.d.ts

@@ -35,5 +35,5 @@ export declare class EntityGHFeature extends Entity {
      * provisioned with old or new code.
      */
     keepUntrackedFiles(_tfOp: string): Promise<void>;
-    getFileContentFromProvider(org: string, url: string): Promise<string>;
+    getFileContentFromProvider(org: string, repo: string, branch: string, path: string): Promise<string>;
 }

package/build/packages/gh_provisioner/src/tp_bridge.d.ts

@@ -15,3 +15,4 @@ export declare function buildContext(entity: Entity, reuseExistingProjectOverrid
     reuseExistingProject: boolean;
     importMode: boolean;
 };
+export declare function calculateTFStatePath(entity: Entity, backend: any): string;

package/build/packages/operator/index.d.ts

@@ -5,3 +5,4 @@ export declare function isImportMode(): boolean;
 export declare function isImportModeSkipPlan(): boolean;
 export declare function isObserveModeActive(): boolean;
 export declare function runOperator(opts: any): void;
+export declare function filterImplementedKinds(kindList: string[]): string[];

package/build/packages/operator/src/connection.d.ts

@@ -0,0 +1,5 @@
+import * as client from '@kubernetes/client-node';
+export declare function getConnection(): Promise<{
+    kc: client.KubeConfig;
+    opts: any;
+}>;

package/build/packages/operator/src/ctl.d.ts

@@ -13,10 +13,8 @@ export declare function writeStatus(kind: string, namespace: string, item: any):
 export declare function writeFinalizer(kind: string, namespace: string, item: any): Promise<any>;
 export declare function listItems(kind: string, namespace: string, kc: any, opts: any): Promise<Array<any>>;
 export declare function observeList(kind: string, namespace: string, revision: string, kc: any, opts: any): AsyncGenerator<any, void, unknown>;
-export declare function getConnection(): Promise<{
-    kc: client.KubeConfig;
-    opts: any;
-}>;
+export { getConnection } from './connection';
+export { getCurrentConcurrentApiCalls, getPendingApiCalls, getPeakConcurrentApiCalls, } from './utils/api-read-limiter';
 export declare function upsertResult(namespace: string, item: any, result: string, exitCode?: number, existingRetryCount?: number): Promise<any>;
 export declare function deleteSecret(secretName: string, namespace: string): Promise<any>;
 export declare function upsertSecret(namespace: string, secret: any): Promise<void>;

package/build/packages/operator/src/diagnosticErrors.d.ts

@@ -0,0 +1,2 @@
+export declare function incrementDiagnosticErrorCount(): void;
+export declare function getDiagnosticErrorCount(): number;

package/build/packages/operator/src/processItem.diagnostic.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="node" />
+import { WorkItem } from './definitions';
+export declare function loopDiagnosticFile(queue: WorkItem[], maxSlots: number): void;
+/** @internal for testing */
+export declare function getDiagnosticTimer(): NodeJS.Timeout | undefined;

package/build/packages/operator/src/utils/api-read-limiter.d.ts

@@ -0,0 +1,10 @@
+import { Semaphore } from './semaphore';
+export declare function computeMaxConcurrentApiCalls(rawMaxConcurrent: string | undefined, rawMaxSlots: string | undefined): number;
+export declare const apiReadSemaphore: Semaphore;
+export declare function withApiReadSlot<T>(read: () => Promise<T>): Promise<T>;
+export declare function getPeakConcurrentApiCalls(): number;
+export declare function getCurrentConcurrentApiCalls(): number;
+export declare function getPendingApiCalls(): number;
+export declare function getMaxConcurrentApiCalls(): number;
+export declare function getTotalApiAcquisitions(): number;
+export declare function getTimesSemaphoreSaturated(): number;

package/build/packages/operator/src/utils/semaphore.d.ts

@@ -0,0 +1,15 @@
+export declare class Semaphore {
+    private readonly _max;
+    private _current;
+    private readonly _queue;
+    private _totalAcquisitions;
+    private _timesSaturated;
+    constructor(max: number);
+    get max(): number;
+    get current(): number;
+    get waiting(): number;
+    get totalAcquisitions(): number;
+    get timesSaturated(): number;
+    acquire(): Promise<void>;
+    release(): void;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firestartr/cli",
-  "version": "2.7.0-snapshot-21",
+  "version": "2.7.0-snapshot-22",
   "private": false,
   "description": "Commandline tool",
   "main": "build/main.js",