@pipeline-builder/api-core 5.2.2 → 5.2.4

package/lib/services/quota-increment-batcher.d.ts

@@ -0,0 +1,81 @@
+/**
+ * Per-pod aggregation of METERING increments (apiCalls / automationCalls).
+ *
+ * Metering used to be one POST to the quota service per successful request,
+ * and each POST was a conditional `findOneAndUpdate` on the org's hot document,
+ * a team counter update, a ledger insert and a threshold check. At the request
+ * rates the meter exists for, the quota service was doing more work than the
+ * request it was counting.
+ *
+ * Increments are now summed in memory per (org, quota type) and, inside that,
+ * per attribution group (principal, key, route pattern, pipeline), then sent
+ * every `QUOTA_INCREMENT_FLUSH_MS` (default 1000) as ONE metered increment per
+ * (org, type) carrying its groups. The quota service applies the sum atomically
+ * against the (pooled) cap with partial admission: the counter moves by as much
+ * as fits and never past the limit, and the rest is recorded as `overshoot` in
+ * the ledger (api/quota `incrementMetered`).
+ *
+ * Guarantees, and their limits:
+ *  - the counter may LAG real usage by up to one flush interval (plus the HTTP
+ *    hop); enforcement is unaffected beyond that lag, because the counter
+ *    still never exceeds the cap;
+ *  - the buffer is bounded (`QUOTA_INCREMENT_BUFFER_MAX` attribution groups).
+ *    Reaching it flushes immediately; only while a flush is already in flight
+ *    is a NEW group dropped, counted on `quota_increment_dropped_total`;
+ *  - a flush that fails is counted on `quota_increment_failures_total` (the
+ *    QuotaIncrementFailures alert) and NOT retried: the POST may have landed,
+ *    and a retry would double-count;
+ *  - graceful shutdown flushes: `runServer` awaits {@link flushQuotaIncrements}
+ *    after the HTTP server has drained, and a SIGTERM flush covers processes
+ *    with their own shutdown path.
+ *
+ * Reservations (aiCalls, pipelines, plugins …) never come through here — they
+ * need the synchronous answer and keep `reserveQuota`.
+ */
+import type { QuotaService, QuotaUsageAttribution } from './quota.js';
+import type { QuotaType } from '../types/common.js';
+/** Flush interval (ms). `0` disables batching: every increment is sent on its own. */
+export declare const QUOTA_INCREMENT_FLUSH_MS: number;
+/** Most attribution groups held between flushes, across every org. */
+export declare const QUOTA_INCREMENT_BUFFER_MAX: number;
+/** Most groups one metered increment carries (the quota service enforces the same bound). */
+export declare const MAX_GROUPS_PER_INCREMENT = 500;
+/** One attribution group of a metered increment: who spent `amount` units. */
+export interface MeteredIncrementGroup {
+    amount: number;
+    attribution?: Omit<QuotaUsageAttribution, 'requestId'>;
+}
+/** Sends one (org, type)'s groups. Resolves when the quota service answered (or not). */
+export type MeteredIncrementSender = (orgId: string, quotaType: QuotaType, groups: MeteredIncrementGroup[]) => Promise<void>;
+/** A per-pod increment aggregator over one sender. Exported for tests. */
+export declare class QuotaIncrementBatcher {
+    private readonly send;
+    private readonly opts;
+    private buffer;
+    private groupCount;
+    private inFlight;
+    private timer;
+    constructor(send: MeteredIncrementSender, opts?: {
+        flushMs: number;
+        maxGroups: number;
+    });
+    /** Units waiting for the next flush (tests, metrics). */
+    pendingUnits(): number;
+    /** Count `amount` units for (org, type). Never throws, never awaits. */
+    add(orgId: string, quotaType: QuotaType, attribution?: QuotaUsageAttribution, amount?: number): void;
+    /** Send everything buffered now. Resolves once every POST has settled. */
+    flush(): Promise<void>;
+    /** Stop the timer (tests; process teardown). Buffered units stay until {@link flush}. */
+    stop(): void;
+    private arm;
+    private dispatch;
+    private sendSafely;
+}
+/** The batcher feeding `quotaService.incrementMetered`. */
+export declare function batcherFor(quotaService: QuotaService, sender: MeteredIncrementSender): QuotaIncrementBatcher;
+/**
+ * Flush every pod-local batcher. Called by `runServer` once the HTTP server has
+ * stopped accepting and in-flight requests have finished (their `finish`
+ * meters are then all buffered). Never rejects.
+ */
+export declare function flushQuotaIncrements(): Promise<void>;

package/lib/services/quota-increment-batcher.js

@@ -0,0 +1,166 @@
+// Copyright 2026 Pipeline Builder Contributors
+// SPDX-License-Identifier: Apache-2.0
+import { runConcurrent } from '../utils/concurrency.js';
+import { envInt } from '../utils/env.js';
+import { createLogger } from '../utils/logger.js';
+import { emitCounter } from '../utils/metric-emitter.js';
+const logger = createLogger('quota-increment-batcher');
+/** Flush interval (ms). `0` disables batching: every increment is sent on its own. */
+export const QUOTA_INCREMENT_FLUSH_MS = envInt('QUOTA_INCREMENT_FLUSH_MS', 1000, { min: 0, max: 60_000 });
+/** Most attribution groups held between flushes, across every org. */
+export const QUOTA_INCREMENT_BUFFER_MAX = envInt('QUOTA_INCREMENT_BUFFER_MAX', 10_000, { min: 1 });
+/** Most groups one metered increment carries (the quota service enforces the same bound). */
+export const MAX_GROUPS_PER_INCREMENT = 500;
+/** Concurrent POSTs per flush. */
+const FLUSH_CONCURRENCY = 8;
+/** The attribution a group aggregates on — everything but the per-request id. */
+function groupAttribution(a) {
+    if (!a)
+        return undefined;
+    const { requestId: _requestId, ...rest } = a;
+    return Object.keys(rest).length > 0 ? rest : undefined;
+}
+function groupKey(a) {
+    return a ? JSON.stringify([a.principalType, a.principalId, a.keyId, a.route, a.pipelineId]) : '';
+}
+/** A per-pod increment aggregator over one sender. Exported for tests. */
+export class QuotaIncrementBatcher {
+    send;
+    opts;
+    buffer = new Map();
+    groupCount = 0;
+    inFlight = null;
+    timer = null;
+    constructor(send, opts = { flushMs: QUOTA_INCREMENT_FLUSH_MS, maxGroups: QUOTA_INCREMENT_BUFFER_MAX }) {
+        this.send = send;
+        this.opts = opts;
+    }
+    /** Units waiting for the next flush (tests, metrics). */
+    pendingUnits() {
+        let n = 0;
+        for (const s of this.buffer.values())
+            for (const g of s.groups.values())
+                n += g.amount;
+        return n;
+    }
+    /** Count `amount` units for (org, type). Never throws, never awaits. */
+    add(orgId, quotaType, attribution, amount = 1) {
+        const attr = groupAttribution(attribution);
+        if (this.opts.flushMs === 0) {
+            this.dispatch(orgId, quotaType, [{ amount, ...(attr && { attribution: attr }) }]);
+            return;
+        }
+        const seriesKey = `${orgId}\u0000${quotaType}`;
+        const key = groupKey(attr);
+        const existing = this.buffer.get(seriesKey)?.groups.get(key);
+        if (existing) {
+            existing.amount += amount;
+            return;
+        }
+        if (this.groupCount >= this.opts.maxGroups) {
+            if (this.inFlight) {
+                emitCounter('quota_increment_dropped_total', { quotaType }, amount);
+                return;
+            }
+            void this.flush();
+        }
+        let series = this.buffer.get(seriesKey);
+        if (!series) {
+            series = { orgId, quotaType, groups: new Map() };
+            this.buffer.set(seriesKey, series);
+        }
+        series.groups.set(key, { amount, ...(attr && { attribution: attr }) });
+        this.groupCount += 1;
+        this.arm();
+    }
+    /** Send everything buffered now. Resolves once every POST has settled. */
+    async flush() {
+        if (this.buffer.size === 0)
+            return this.inFlight ?? undefined;
+        const batch = [...this.buffer.values()];
+        this.buffer = new Map();
+        this.groupCount = 0;
+        const previous = this.inFlight;
+        const run = (async () => {
+            await runConcurrent(batch, FLUSH_CONCURRENCY, async (series) => {
+                const groups = [...series.groups.values()];
+                for (let i = 0; i < groups.length; i += MAX_GROUPS_PER_INCREMENT) {
+                    await this.sendSafely(series.orgId, series.quotaType, groups.slice(i, i + MAX_GROUPS_PER_INCREMENT));
+                }
+            });
+            if (previous)
+                await previous;
+        })();
+        this.inFlight = run;
+        try {
+            await run;
+        }
+        finally {
+            if (this.inFlight === run)
+                this.inFlight = null;
+        }
+    }
+    /** Stop the timer (tests; process teardown). Buffered units stay until {@link flush}. */
+    stop() {
+        if (this.timer)
+            clearInterval(this.timer);
+        this.timer = null;
+    }
+    arm() {
+        if (this.timer)
+            return;
+        this.timer = setInterval(() => { void this.flush(); }, this.opts.flushMs);
+        this.timer.unref?.();
+        armShutdownFlush();
+    }
+    dispatch(orgId, quotaType, groups) {
+        void this.sendSafely(orgId, quotaType, groups);
+    }
+    async sendSafely(orgId, quotaType, groups) {
+        try {
+            await this.send(orgId, quotaType, groups);
+        }
+        catch (err) {
+            logger.warn('Metered quota increment failed', { orgId, quotaType, error: String(err) });
+            emitCounter('quota_increment_failures_total', { reason: 'exception', quotaType });
+        }
+    }
+}
+/** One batcher per quota client, so a test's fake client never shares a buffer. */
+const batchers = new WeakMap();
+/** The live batchers, for the shutdown flush (a WeakMap cannot be iterated). */
+const live = new Set();
+/** The batcher feeding `quotaService.incrementMetered`. */
+export function batcherFor(quotaService, sender) {
+    let b = batchers.get(quotaService);
+    if (!b) {
+        b = new QuotaIncrementBatcher(sender);
+        batchers.set(quotaService, b);
+        live.add(b);
+    }
+    return b;
+}
+/**
+ * Flush every pod-local batcher. Called by `runServer` once the HTTP server has
+ * stopped accepting and in-flight requests have finished (their `finish`
+ * meters are then all buffered). Never rejects.
+ */
+export async function flushQuotaIncrements() {
+    await Promise.all([...live].map((b) => b.flush().catch(() => undefined)));
+}
+let shutdownArmed = false;
+/**
+ * SIGTERM flush for processes that don't shut down through `runServer`.
+ *
+ * Armed only when the process ALREADY handles SIGTERM (every service registers
+ * its shutdown at startup, before the first request is metered): adding the
+ * first listener would switch off Node's default exit-on-SIGTERM for a process
+ * that relied on it (a CLI, a test worker).
+ */
+function armShutdownFlush() {
+    if (shutdownArmed || process.listenerCount('SIGTERM') === 0)
+        return;
+    shutdownArmed = true;
+    process.once('SIGTERM', () => { void flushQuotaIncrements(); });
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicXVvdGEtaW5jcmVtZW50LWJhdGNoZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvc2VydmljZXMvcXVvdGEtaW5jcmVtZW50LWJhdGNoZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsK0NBQStDO0FBQy9DLHNDQUFzQztBQXVDdEMsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQ3hELE9BQU8sRUFBRSxNQUFNLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUN6QyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDbEQsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLDRCQUE0QixDQUFDO0FBRXpELE1BQU0sTUFBTSxHQUFHLFlBQVksQ0FBQyx5QkFBeUIsQ0FBQyxDQUFDO0FBRXZELHNGQUFzRjtBQUN0RixNQUFNLENBQUMsTUFBTSx3QkFBd0IsR0FBRyxNQUFNLENBQUMsMEJBQTBCLEVBQUUsSUFBSSxFQUFFLEVBQUUsR0FBRyxFQUFFLENBQUMsRUFBRSxHQUFHLEVBQUUsTUFBTSxFQUFFLENBQUMsQ0FBQztBQUUxRyxzRUFBc0U7QUFDdEUsTUFBTSxDQUFDLE1BQU0sMEJBQTBCLEdBQUcsTUFBTSxDQUFDLDRCQUE0QixFQUFFLE1BQU0sRUFBRSxFQUFFLEdBQUcsRUFBRSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0FBRW5HLDZGQUE2RjtBQUM3RixNQUFNLENBQUMsTUFBTSx3QkFBd0IsR0FBRyxHQUFHLENBQUM7QUFFNUMsa0NBQWtDO0FBQ2xDLE1BQU0saUJBQWlCLEdBQUcsQ0FBQyxDQUFDO0FBaUI1QixpRkFBaUY7QUFDakYsU0FBUyxnQkFBZ0IsQ0FBQyxDQUFvQztJQUM1RCxJQUFJLENBQUMsQ0FBQztRQUFFLE9BQU8sU0FBUyxDQUFDO0lBQ3pCLE1BQU0sRUFBRSxTQUFTLEVBQUUsVUFBVSxFQUFFLEdBQUcsSUFBSSxFQUFFLEdBQUcsQ0FBQyxDQUFDO0lBQzdDLE9BQU8sTUFBTSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQztBQUN6RCxDQUFDO0FBRUQsU0FBUyxRQUFRLENBQUMsQ0FBdUM7SUFDdkQsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLENBQUMsYUFBYSxFQUFFLENBQUMsQ0FBQyxXQUFXLEVBQUUsQ0FBQyxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUMsS0FBSyxFQUFFLENBQUMsQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7QUFDbkcsQ0FBQztBQUVELDBFQUEwRTtBQUMxRSxNQUFNLE9BQU8scUJBQXFCO0lBT2IsSUFBSTtJQUNKLElBQUk7SUFQZixNQUFNLEdBQUcsSUFBSSxHQUFHLEVBQXlCLENBQUM7SUFDMUMsVUFBVSxHQUFHLENBQUMsQ0FBQztJQUNmLFFBQVEsR0FBeUIsSUFBSSxDQUFDO0lBQ3RDLEtBQUssR0FBMEIsSUFBSSxDQUFDO0lBRTVDLFlBQ21CLElBQTRCLEVBQzVCLElBQUksR0FBMkMsRUFBRSxPQUFPLEVBQUUsd0JBQXdCLEVBQUUsU0FBUyxFQUFFLDBCQUEwQixFQUFFO29CQUQzSCxJQUFJO29CQUNKLElBQUk7SUFDcEIsQ0FBQztJQUVKLHlEQUF5RDtJQUN6RCxZQUFZO1FBQ1YsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1FBQ1YsS0FBSyxNQUFNLENBQUMsSUFBSSxJQUFJLENBQUMsTUFBTSxDQUFDLE1BQU0sRUFBRTtZQUFFLEtBQUssTUFBTSxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sQ0FBQyxNQUFNLEVBQUU7Z0JBQUUsQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFNLENBQUM7UUFDdkYsT0FBTyxDQUFDLENBQUM7SUFDWCxDQUFDO0lBRUQsd0VBQXdFO0lBQ3hFLEdBQUcsQ0FBQyxLQUFhLEVBQUUsU0FBb0IsRUFBRSxXQUFtQyxFQUFFLE1BQU0sR0FBVyxDQUFDO1FBQzlGLE1BQU0sSUFBSSxHQUFHLGdCQUFnQixDQUFDLFdBQVcsQ0FBQyxDQUFDO1FBQzNDLElBQUksSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLEtBQUssQ0FBQyxFQUFFLENBQUM7WUFDNUIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxLQUFLLEVBQUUsU0FBUyxFQUFFLENBQUMsRUFBRSxNQUFNLEVBQUUsR0FBRyxDQUFDLElBQUksSUFBSSxFQUFFLFdBQVcsRUFBRSxJQUFJLEVBQUUsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDO1lBQ2xGLE9BQU87UUFDVCxDQUFDO1FBQ0QsTUFBTSxTQUFTLEdBQUcsR0FBRyxLQUFLLFNBQVMsU0FBUyxFQUFFLENBQUM7UUFDL0MsTUFBTSxHQUFHLEdBQUcsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDO1FBQzNCLE1BQU0sUUFBUSxHQUFHLElBQUksQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLFNBQVMsQ0FBQyxFQUFFLE1BQU0sQ0FBQyxHQUFHLENBQUMsR0FBRyxDQUFDLENBQUM7UUFDN0QsSUFBSSxRQUFRLEVBQUUsQ0FBQztZQUNiLFFBQVEsQ0FBQyxNQUFNLElBQUksTUFBTSxDQUFDO1lBQzFCLE9BQU87UUFDVCxDQUFDO1FBQ0QsSUFBSSxJQUFJLENBQUMsVUFBVSxJQUFJLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLENBQUM7WUFDM0MsSUFBSSxJQUFJLENBQUMsUUFBUSxFQUFFLENBQUM7Z0JBQ2xCLFdBQVcsQ0FBQywrQkFBK0IsRUFBRSxFQUFFLFNBQVMsRUFBRSxFQUFFLE1BQU0sQ0FBQyxDQUFDO2dCQUNwRSxPQUFPO1lBQ1QsQ0FBQztZQUNELEtBQUssSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDO1FBQ3BCLENBQUM7UUFDRCxJQUFJLE1BQU0sR0FBRyxJQUFJLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxTQUFTLENBQUMsQ0FBQztRQUN4QyxJQUFJLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDWixNQUFNLEdBQUcsRUFBRSxLQUFLLEVBQUUsU0FBUyxFQUFFLE1BQU0sRUFBRSxJQUFJLEdBQUcsRUFBRSxFQUFFLENBQUM7WUFDakQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsU0FBUyxFQUFFLE1BQU0sQ0FBQyxDQUFDO1FBQ3JDLENBQUM7UUFDRCxNQUFNLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxHQUFHLEVBQUUsRUFBRSxNQUFNLEVBQUUsR0FBRyxDQUFDLElBQUksSUFBSSxFQUFFLFdBQVcsRUFBRSxJQUFJLEVBQUUsQ0FBQyxFQUFFLENBQUMsQ0FBQztRQUN2RSxJQUFJLENBQUMsVUFBVSxJQUFJLENBQUMsQ0FBQztRQUNyQixJQUFJLENBQUMsR0FBRyxFQUFFLENBQUM7SUFDYixDQUFDO0lBRUQsMEVBQTBFO0lBQzFFLEtBQUssQ0FBQyxLQUFLO1FBQ1QsSUFBSSxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksS0FBSyxDQUFDO1lBQUUsT0FBTyxJQUFJLENBQUMsUUFBUSxJQUFJLFNBQVMsQ0FBQztRQUM5RCxNQUFNLEtBQUssR0FBRyxDQUFDLEdBQUcsSUFBSSxDQUFDLE1BQU0sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDO1FBQ3hDLElBQUksQ0FBQyxNQUFNLEdBQUcsSUFBSSxHQUFHLEVBQUUsQ0FBQztRQUN4QixJQUFJLENBQUMsVUFBVSxHQUFHLENBQUMsQ0FBQztRQUNwQixNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsUUFBUSxDQUFDO1FBQy9CLE1BQU0sR0FBRyxHQUFHLENBQUMsS0FBSyxJQUFJLEVBQUU7WUFDdEIsTUFBTSxhQUFhLENBQUMsS0FBSyxFQUFFLGlCQUFpQixFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsRUFBRTtnQkFDN0QsTUFBTSxNQUFNLEdBQUcsQ0FBQyxHQUFHLE1BQU0sQ0FBQyxNQUFNLENBQUMsTUFBTSxFQUFFLENBQUMsQ0FBQztnQkFDM0MsS0FBSyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxHQUFHLE1BQU0sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxJQUFJLHdCQUF3QixFQUFFLENBQUM7b0JBQ2pFLE1BQU0sSUFBSSxDQUFDLFVBQVUsQ0FBQyxNQUFNLENBQUMsS0FBSyxFQUFFLE1BQU0sQ0FBQyxTQUFTLEVBQUUsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDLEVBQUUsQ0FBQyxHQUFHLHdCQUF3QixDQUFDLENBQUMsQ0FBQztnQkFDdkcsQ0FBQztZQUNILENBQUMsQ0FBQyxDQUFDO1lBQ0gsSUFBSSxRQUFRO2dCQUFFLE1BQU0sUUFBUSxDQUFDO1FBQy9CLENBQUMsQ0FBQyxFQUFFLENBQUM7UUFDTCxJQUFJLENBQUMsUUFBUSxHQUFHLEdBQUcsQ0FBQztRQUNwQixJQUFJLENBQUM7WUFDSCxNQUFNLEdBQUcsQ0FBQztRQUNaLENBQUM7Z0JBQVMsQ0FBQztZQUNULElBQUksSUFBSSxDQUFDLFFBQVEsS0FBSyxHQUFHO2dCQUFFLElBQUksQ0FBQyxRQUFRLEdBQUcsSUFBSSxDQUFDO1FBQ2xELENBQUM7SUFDSCxDQUFDO0lBRUQseUZBQXlGO0lBQ3pGLElBQUk7UUFDRixJQUFJLElBQUksQ0FBQyxLQUFLO1lBQUUsYUFBYSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUMxQyxJQUFJLENBQUMsS0FBSyxHQUFHLElBQUksQ0FBQztJQUNwQixDQUFDO0lBRU8sR0FBRztRQUNULElBQUksSUFBSSxDQUFDLEtBQUs7WUFBRSxPQUFPO1FBQ3ZCLElBQUksQ0FBQyxLQUFLLEdBQUcsV0FBVyxDQUFDLEdBQUcsRUFBRSxHQUFHLEtBQUssSUFBSSxDQUFDLEtBQUssRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDMUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLEVBQUUsRUFBRSxDQUFDO1FBQ3JCLGdCQUFnQixFQUFFLENBQUM7SUFDckIsQ0FBQztJQUVPLFFBQVEsQ0FBQyxLQUFhLEVBQUUsU0FBb0IsRUFBRSxNQUErQjtRQUNuRixLQUFLLElBQUksQ0FBQyxVQUFVLENBQUMsS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLENBQUMsQ0FBQztJQUNqRCxDQUFDO0lBRU8sS0FBSyxDQUFDLFVBQVUsQ0FBQyxLQUFhLEVBQUUsU0FBb0IsRUFBRSxNQUErQjtRQUMzRixJQUFJLENBQUM7WUFDSCxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLENBQUMsQ0FBQztRQUM1QyxDQUFDO1FBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztZQUNiLE1BQU0sQ0FBQyxJQUFJLENBQUMsZ0NBQWdDLEVBQUUsRUFBRSxLQUFLLEVBQUUsU0FBUyxFQUFFLEtBQUssRUFBRSxNQUFNLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ3hGLFdBQVcsQ0FBQyxnQ0FBZ0MsRUFBRSxFQUFFLE1BQU0sRUFBRSxXQUFXLEVBQUUsU0FBUyxFQUFFLENBQUMsQ0FBQztRQUNwRixDQUFDO0lBQ0gsQ0FBQztDQUNGO0FBRUQsbUZBQW1GO0FBQ25GLE1BQU0sUUFBUSxHQUFHLElBQUksT0FBTyxFQUF1QyxDQUFDO0FBRXBFLGdGQUFnRjtBQUNoRixNQUFNLElBQUksR0FBRyxJQUFJLEdBQUcsRUFBeUIsQ0FBQztBQUU5QywyREFBMkQ7QUFDM0QsTUFBTSxVQUFVLFVBQVUsQ0FBQyxZQUEwQixFQUFFLE1BQThCO0lBQ25GLElBQUksQ0FBQyxHQUFHLFFBQVEsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUM7SUFDbkMsSUFBSSxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQ1AsQ0FBQyxHQUFHLElBQUkscUJBQXFCLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDdEMsUUFBUSxDQUFDLEdBQUcsQ0FBQyxZQUFZLEVBQUUsQ0FBQyxDQUFDLENBQUM7UUFDOUIsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUNkLENBQUM7SUFDRCxPQUFPLENBQUMsQ0FBQztBQUNYLENBQUM7QUFFRDs7OztHQUlHO0FBQ0gsTUFBTSxDQUFDLEtBQUssVUFBVSxvQkFBb0I7SUFDeEMsTUFBTSxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUMsR0FBRyxJQUFJLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxLQUFLLEVBQUUsQ0FBQyxLQUFLLENBQUMsR0FBRyxFQUFFLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQzVFLENBQUM7QUFFRCxJQUFJLGFBQWEsR0FBRyxLQUFLLENBQUM7QUFFMUI7Ozs7Ozs7R0FPRztBQUNILFNBQVMsZ0JBQWdCO0lBQ3ZCLElBQUksYUFBYSxJQUFJLE9BQU8sQ0FBQyxhQUFhLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQztRQUFFLE9BQU87SUFDcEUsYUFBYSxHQUFHLElBQUksQ0FBQztJQUNyQixPQUFPLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxHQUFHLEVBQUUsR0FBRyxLQUFLLG9CQUFvQixFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztBQUNsRSxDQUFDIiwic291cmNlc0NvbnRlbnQiOlsiLy8gQ29weXJpZ2h0IDIwMjYgUGlwZWxpbmUgQnVpbGRlciBDb250cmlidXRvcnNcbi8vIFNQRFgtTGljZW5zZS1JZGVudGlmaWVyOiBBcGFjaGUtMi4wXG5cbi8qKlxuICogUGVyLXBvZCBhZ2dyZWdhdGlvbiBvZiBNRVRFUklORyBpbmNyZW1lbnRzIChhcGlDYWxscyAvIGF1dG9tYXRpb25DYWxscykuXG4gKlxuICogTWV0ZXJpbmcgdXNlZCB0byBiZSBvbmUgUE9TVCB0byB0aGUgcXVvdGEgc2VydmljZSBwZXIgc3VjY2Vzc2Z1bCByZXF1ZXN0LFxuICogYW5kIGVhY2ggUE9TVCB3YXMgYSBjb25kaXRpb25hbCBgZmluZE9uZUFuZFVwZGF0ZWAgb24gdGhlIG9yZydzIGhvdCBkb2N1bWVudCxcbiAqIGEgdGVhbSBjb3VudGVyIHVwZGF0ZSwgYSBsZWRnZXIgaW5zZXJ0IGFuZCBhIHRocmVzaG9sZCBjaGVjay4gQXQgdGhlIHJlcXVlc3RcbiAqIHJhdGVzIHRoZSBtZXRlciBleGlzdHMgZm9yLCB0aGUgcXVvdGEgc2VydmljZSB3YXMgZG9pbmcgbW9yZSB3b3JrIHRoYW4gdGhlXG4gKiByZXF1ZXN0IGl0IHdhcyBjb3VudGluZy5cbiAqXG4gKiBJbmNyZW1lbnRzIGFyZSBub3cgc3VtbWVkIGluIG1lbW9yeSBwZXIgKG9yZywgcXVvdGEgdHlwZSkgYW5kLCBpbnNpZGUgdGhhdCxcbiAqIHBlciBhdHRyaWJ1dGlvbiBncm91cCAocHJpbmNpcGFsLCBrZXksIHJvdXRlIHBhdHRlcm4sIHBpcGVsaW5lKSwgdGhlbiBzZW50XG4gKiBldmVyeSBgUVVPVEFfSU5DUkVNRU5UX0ZMVVNIX01TYCAoZGVmYXVsdCAxMDAwKSBhcyBPTkUgbWV0ZXJlZCBpbmNyZW1lbnQgcGVyXG4gKiAob3JnLCB0eXBlKSBjYXJyeWluZyBpdHMgZ3JvdXBzLiBUaGUgcXVvdGEgc2VydmljZSBhcHBsaWVzIHRoZSBzdW0gYXRvbWljYWxseVxuICogYWdhaW5zdCB0aGUgKHBvb2xlZCkgY2FwIHdpdGggcGFydGlhbCBhZG1pc3Npb246IHRoZSBjb3VudGVyIG1vdmVzIGJ5IGFzIG11Y2hcbiAqIGFzIGZpdHMgYW5kIG5ldmVyIHBhc3QgdGhlIGxpbWl0LCBhbmQgdGhlIHJlc3QgaXMgcmVjb3JkZWQgYXMgYG92ZXJzaG9vdGAgaW5cbiAqIHRoZSBsZWRnZXIgKGFwaS9xdW90YSBgaW5jcmVtZW50TWV0ZXJlZGApLlxuICpcbiAqIEd1YXJhbnRlZXMsIGFuZCB0aGVpciBsaW1pdHM6XG4gKiAgLSB0aGUgY291bnRlciBtYXkgTEFHIHJlYWwgdXNhZ2UgYnkgdXAgdG8gb25lIGZsdXNoIGludGVydmFsIChwbHVzIHRoZSBIVFRQXG4gKiAgICBob3ApOyBlbmZvcmNlbWVudCBpcyB1bmFmZmVjdGVkIGJleW9uZCB0aGF0IGxhZywgYmVjYXVzZSB0aGUgY291bnRlclxuICogICAgc3RpbGwgbmV2ZXIgZXhjZWVkcyB0aGUgY2FwO1xuICogIC0gdGhlIGJ1ZmZlciBpcyBib3VuZGVkIChgUVVPVEFfSU5DUkVNRU5UX0JVRkZFUl9NQVhgIGF0dHJpYnV0aW9uIGdyb3VwcykuXG4gKiAgICBSZWFjaGluZyBpdCBmbHVzaGVzIGltbWVkaWF0ZWx5OyBvbmx5IHdoaWxlIGEgZmx1c2ggaXMgYWxyZWFkeSBpbiBmbGlnaHRcbiAqICAgIGlzIGEgTkVXIGdyb3VwIGRyb3BwZWQsIGNvdW50ZWQgb24gYHF1b3RhX2luY3JlbWVudF9kcm9wcGVkX3RvdGFsYDtcbiAqICAtIGEgZmx1c2ggdGhhdCBmYWlscyBpcyBjb3VudGVkIG9uIGBxdW90YV9pbmNyZW1lbnRfZmFpbHVyZXNfdG90YWxgICh0aGVcbiAqICAgIFF1b3RhSW5jcmVtZW50RmFpbHVyZXMgYWxlcnQpIGFuZCBOT1QgcmV0cmllZDogdGhlIFBPU1QgbWF5IGhhdmUgbGFuZGVkLFxuICogICAgYW5kIGEgcmV0cnkgd291bGQgZG91YmxlLWNvdW50O1xuICogIC0gZ3JhY2VmdWwgc2h1dGRvd24gZmx1c2hlczogYHJ1blNlcnZlcmAgYXdhaXRzIHtAbGluayBmbHVzaFF1b3RhSW5jcmVtZW50c31cbiAqICAgIGFmdGVyIHRoZSBIVFRQIHNlcnZlciBoYXMgZHJhaW5lZCwgYW5kIGEgU0lHVEVSTSBmbHVzaCBjb3ZlcnMgcHJvY2Vzc2VzXG4gKiAgICB3aXRoIHRoZWlyIG93biBzaHV0ZG93biBwYXRoLlxuICpcbiAqIFJlc2VydmF0aW9ucyAoYWlDYWxscywgcGlwZWxpbmVzLCBwbHVnaW5zIOKApikgbmV2ZXIgY29tZSB0aHJvdWdoIGhlcmUg4oCUIHRoZXlcbiAqIG5lZWQgdGhlIHN5bmNocm9ub3VzIGFuc3dlciBhbmQga2VlcCBgcmVzZXJ2ZVF1b3RhYC5cbiAqL1xuXG5pbXBvcnQgdHlwZSB7IFF1b3RhU2VydmljZSwgUXVvdGFVc2FnZUF0dHJpYnV0aW9uIH0gZnJvbSAnLi9xdW90YS5qcyc7XG5pbXBvcnQgdHlwZSB7IFF1b3RhVHlwZSB9IGZyb20gJy4uL3R5cGVzL2NvbW1vbi5qcyc7XG5pbXBvcnQgeyBydW5Db25jdXJyZW50IH0gZnJvbSAnLi4vdXRpbHMvY29uY3VycmVuY3kuanMnO1xuaW1wb3J0IHsgZW52SW50IH0gZnJvbSAnLi4vdXRpbHMvZW52LmpzJztcbmltcG9ydCB7IGNyZWF0ZUxvZ2dlciB9IGZyb20gJy4uL3V0aWxzL2xvZ2dlci5qcyc7XG5pbXBvcnQgeyBlbWl0Q291bnRlciB9IGZyb20gJy4uL3V0aWxzL21ldHJpYy1lbWl0dGVyLmpzJztcblxuY29uc3QgbG9nZ2VyID0gY3JlYXRlTG9nZ2VyKCdxdW90YS1pbmNyZW1lbnQtYmF0Y2hlcicpO1xuXG4vKiogRmx1c2ggaW50ZXJ2YWwgKG1zKS4gYDBgIGRpc2FibGVzIGJhdGNoaW5nOiBldmVyeSBpbmNyZW1lbnQgaXMgc2VudCBvbiBpdHMgb3duLiAqL1xuZXhwb3J0IGNvbnN0IFFVT1RBX0lOQ1JFTUVOVF9GTFVTSF9NUyA9IGVudkludCgnUVVPVEFfSU5DUkVNRU5UX0ZMVVNIX01TJywgMTAwMCwgeyBtaW46IDAsIG1heDogNjBfMDAwIH0pO1xuXG4vKiogTW9zdCBhdHRyaWJ1dGlvbiBncm91cHMgaGVsZCBiZXR3ZWVuIGZsdXNoZXMsIGFjcm9zcyBldmVyeSBvcmcuICovXG5leHBvcnQgY29uc3QgUVVPVEFfSU5DUkVNRU5UX0JVRkZFUl9NQVggPSBlbnZJbnQoJ1FVT1RBX0lOQ1JFTUVOVF9CVUZGRVJfTUFYJywgMTBfMDAwLCB7IG1pbjogMSB9KTtcblxuLyoqIE1vc3QgZ3JvdXBzIG9uZSBtZXRlcmVkIGluY3JlbWVudCBjYXJyaWVzICh0aGUgcXVvdGEgc2VydmljZSBlbmZvcmNlcyB0aGUgc2FtZSBib3VuZCkuICovXG5leHBvcnQgY29uc3QgTUFYX0dST1VQU19QRVJfSU5DUkVNRU5UID0gNTAwO1xuXG4vKiogQ29uY3VycmVudCBQT1NUcyBwZXIgZmx1c2guICovXG5jb25zdCBGTFVTSF9DT05DVVJSRU5DWSA9IDg7XG5cbi8qKiBPbmUgYXR0cmlidXRpb24gZ3JvdXAgb2YgYSBtZXRlcmVkIGluY3JlbWVudDogd2hvIHNwZW50IGBhbW91bnRgIHVuaXRzLiAqL1xuZXhwb3J0IGludGVyZmFjZSBNZXRlcmVkSW5jcmVtZW50R3JvdXAge1xuICBhbW91bnQ6IG51bWJlcjtcbiAgYXR0cmlidXRpb24/OiBPbWl0PFF1b3RhVXNhZ2VBdHRyaWJ1dGlvbiwgJ3JlcXVlc3RJZCc+O1xufVxuXG4vKiogU2VuZHMgb25lIChvcmcsIHR5cGUpJ3MgZ3JvdXBzLiBSZXNvbHZlcyB3aGVuIHRoZSBxdW90YSBzZXJ2aWNlIGFuc3dlcmVkIChvciBub3QpLiAqL1xuZXhwb3J0IHR5cGUgTWV0ZXJlZEluY3JlbWVudFNlbmRlciA9IChvcmdJZDogc3RyaW5nLCBxdW90YVR5cGU6IFF1b3RhVHlwZSwgZ3JvdXBzOiBNZXRlcmVkSW5jcmVtZW50R3JvdXBbXSkgPT4gUHJvbWlzZTx2b2lkPjtcblxuaW50ZXJmYWNlIFBlbmRpbmdTZXJpZXMge1xuICBvcmdJZDogc3RyaW5nO1xuICBxdW90YVR5cGU6IFF1b3RhVHlwZTtcbiAgZ3JvdXBzOiBNYXA8c3RyaW5nLCBNZXRlcmVkSW5jcmVtZW50R3JvdXA+O1xufVxuXG4vKiogVGhlIGF0dHJpYnV0aW9uIGEgZ3JvdXAgYWdncmVnYXRlcyBvbiDigJQgZXZlcnl0aGluZyBidXQgdGhlIHBlci1yZXF1ZXN0IGlkLiAqL1xuZnVuY3Rpb24gZ3JvdXBBdHRyaWJ1dGlvbihhOiBRdW90YVVzYWdlQXR0cmlidXRpb24gfCB1bmRlZmluZWQpOiBNZXRlcmVkSW5jcmVtZW50R3JvdXBbJ2F0dHJpYnV0aW9uJ10ge1xuICBpZiAoIWEpIHJldHVybiB1bmRlZmluZWQ7XG4gIGNvbnN0IHsgcmVxdWVzdElkOiBfcmVxdWVzdElkLCAuLi5yZXN0IH0gPSBhO1xuICByZXR1cm4gT2JqZWN0LmtleXMocmVzdCkubGVuZ3RoID4gMCA/IHJlc3QgOiB1bmRlZmluZWQ7XG59XG5cbmZ1bmN0aW9uIGdyb3VwS2V5KGE6IE1ldGVyZWRJbmNyZW1lbnRHcm91cFsnYXR0cmlidXRpb24nXSk6IHN0cmluZyB7XG4gIHJldHVybiBhID8gSlNPTi5zdHJpbmdpZnkoW2EucHJpbmNpcGFsVHlwZSwgYS5wcmluY2lwYWxJZCwgYS5rZXlJZCwgYS5yb3V0ZSwgYS5waXBlbGluZUlkXSkgOiAnJztcbn1cblxuLyoqIEEgcGVyLXBvZCBpbmNyZW1lbnQgYWdncmVnYXRvciBvdmVyIG9uZSBzZW5kZXIuIEV4cG9ydGVkIGZvciB0ZXN0cy4gKi9cbmV4cG9ydCBjbGFzcyBRdW90YUluY3JlbWVudEJhdGNoZXIge1xuICBwcml2YXRlIGJ1ZmZlciA9IG5ldyBNYXA8c3RyaW5nLCBQZW5kaW5nU2VyaWVzPigpO1xuICBwcml2YXRlIGdyb3VwQ291bnQgPSAwO1xuICBwcml2YXRlIGluRmxpZ2h0OiBQcm9taXNlPHZvaWQ+IHwgbnVsbCA9IG51bGw7XG4gIHByaXZhdGUgdGltZXI6IE5vZGVKUy5UaW1lb3V0IHwgbnVsbCA9IG51bGw7XG5cbiAgY29uc3RydWN0b3IoXG4gICAgcHJpdmF0ZSByZWFkb25seSBzZW5kOiBNZXRlcmVkSW5jcmVtZW50U2VuZGVyLFxuICAgIHByaXZhdGUgcmVhZG9ubHkgb3B0czogeyBmbHVzaE1zOiBudW1iZXI7IG1heEdyb3VwczogbnVtYmVyIH0gPSB7IGZsdXNoTXM6IFFVT1RBX0lOQ1JFTUVOVF9GTFVTSF9NUywgbWF4R3JvdXBzOiBRVU9UQV9JTkNSRU1FTlRfQlVGRkVSX01BWCB9LFxuICApIHt9XG5cbiAgLyoqIFVuaXRzIHdhaXRpbmcgZm9yIHRoZSBuZXh0IGZsdXNoICh0ZXN0cywgbWV0cmljcykuICovXG4gIHBlbmRpbmdVbml0cygpOiBudW1iZXIge1xuICAgIGxldCBuID0gMDtcbiAgICBmb3IgKGNvbnN0IHMgb2YgdGhpcy5idWZmZXIudmFsdWVzKCkpIGZvciAoY29uc3QgZyBvZiBzLmdyb3Vwcy52YWx1ZXMoKSkgbiArPSBnLmFtb3VudDtcbiAgICByZXR1cm4gbjtcbiAgfVxuXG4gIC8qKiBDb3VudCBgYW1vdW50YCB1bml0cyBmb3IgKG9yZywgdHlwZSkuIE5ldmVyIHRocm93cywgbmV2ZXIgYXdhaXRzLiAqL1xuICBhZGQob3JnSWQ6IHN0cmluZywgcXVvdGFUeXBlOiBRdW90YVR5cGUsIGF0dHJpYnV0aW9uPzogUXVvdGFVc2FnZUF0dHJpYnV0aW9uLCBhbW91bnQ6IG51bWJlciA9IDEpOiB2b2lkIHtcbiAgICBjb25zdCBhdHRyID0gZ3JvdXBBdHRyaWJ1dGlvbihhdHRyaWJ1dGlvbik7XG4gICAgaWYgKHRoaXMub3B0cy5mbHVzaE1zID09PSAwKSB7XG4gICAgICB0aGlzLmRpc3BhdGNoKG9yZ0lkLCBxdW90YVR5cGUsIFt7IGFtb3VudCwgLi4uKGF0dHIgJiYgeyBhdHRyaWJ1dGlvbjogYXR0ciB9KSB9XSk7XG4gICAgICByZXR1cm47XG4gICAgfVxuICAgIGNvbnN0IHNlcmllc0tleSA9IGAke29yZ0lkfVxcdTAwMDAke3F1b3RhVHlwZX1gO1xuICAgIGNvbnN0IGtleSA9IGdyb3VwS2V5KGF0dHIpO1xuICAgIGNvbnN0IGV4aXN0aW5nID0gdGhpcy5idWZmZXIuZ2V0KHNlcmllc0tleSk/Lmdyb3Vwcy5nZXQoa2V5KTtcbiAgICBpZiAoZXhpc3RpbmcpIHtcbiAgICAgIGV4aXN0aW5nLmFtb3VudCArPSBhbW91bnQ7XG4gICAgICByZXR1cm47XG4gICAgfVxuICAgIGlmICh0aGlzLmdyb3VwQ291bnQgPj0gdGhpcy5vcHRzLm1heEdyb3Vwcykge1xuICAgICAgaWYgKHRoaXMuaW5GbGlnaHQpIHtcbiAgICAgICAgZW1pdENvdW50ZXIoJ3F1b3RhX2luY3JlbWVudF9kcm9wcGVkX3RvdGFsJywgeyBxdW90YVR5cGUgfSwgYW1vdW50KTtcbiAgICAgICAgcmV0dXJuO1xuICAgICAgfVxuICAgICAgdm9pZCB0aGlzLmZsdXNoKCk7XG4gICAgfVxuICAgIGxldCBzZXJpZXMgPSB0aGlzLmJ1ZmZlci5nZXQoc2VyaWVzS2V5KTtcbiAgICBpZiAoIXNlcmllcykge1xuICAgICAgc2VyaWVzID0geyBvcmdJZCwgcXVvdGFUeXBlLCBncm91cHM6IG5ldyBNYXAoKSB9O1xuICAgICAgdGhpcy5idWZmZXIuc2V0KHNlcmllc0tleSwgc2VyaWVzKTtcbiAgICB9XG4gICAgc2VyaWVzLmdyb3Vwcy5zZXQoa2V5LCB7IGFtb3VudCwgLi4uKGF0dHIgJiYgeyBhdHRyaWJ1dGlvbjogYXR0ciB9KSB9KTtcbiAgICB0aGlzLmdyb3VwQ291bnQgKz0gMTtcbiAgICB0aGlzLmFybSgpO1xuICB9XG5cbiAgLyoqIFNlbmQgZXZlcnl0aGluZyBidWZmZXJlZCBub3cuIFJlc29sdmVzIG9uY2UgZXZlcnkgUE9TVCBoYXMgc2V0dGxlZC4gKi9cbiAgYXN5bmMgZmx1c2goKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgaWYgKHRoaXMuYnVmZmVyLnNpemUgPT09IDApIHJldHVybiB0aGlzLmluRmxpZ2h0ID8/IHVuZGVmaW5lZDtcbiAgICBjb25zdCBiYXRjaCA9IFsuLi50aGlzLmJ1ZmZlci52YWx1ZXMoKV07XG4gICAgdGhpcy5idWZmZXIgPSBuZXcgTWFwKCk7XG4gICAgdGhpcy5ncm91cENvdW50ID0gMDtcbiAgICBjb25zdCBwcmV2aW91cyA9IHRoaXMuaW5GbGlnaHQ7XG4gICAgY29uc3QgcnVuID0gKGFzeW5jICgpID0+IHtcbiAgICAgIGF3YWl0IHJ1bkNvbmN1cnJlbnQoYmF0Y2gsIEZMVVNIX0NPTkNVUlJFTkNZLCBhc3luYyAoc2VyaWVzKSA9PiB7XG4gICAgICAgIGNvbnN0IGdyb3VwcyA9IFsuLi5zZXJpZXMuZ3JvdXBzLnZhbHVlcygpXTtcbiAgICAgICAgZm9yIChsZXQgaSA9IDA7IGkgPCBncm91cHMubGVuZ3RoOyBpICs9IE1BWF9HUk9VUFNfUEVSX0lOQ1JFTUVOVCkge1xuICAgICAgICAgIGF3YWl0IHRoaXMuc2VuZFNhZmVseShzZXJpZXMub3JnSWQsIHNlcmllcy5xdW90YVR5cGUsIGdyb3Vwcy5zbGljZShpLCBpICsgTUFYX0dST1VQU19QRVJfSU5DUkVNRU5UKSk7XG4gICAgICAgIH1cbiAgICAgIH0pO1xuICAgICAgaWYgKHByZXZpb3VzKSBhd2FpdCBwcmV2aW91cztcbiAgICB9KSgpO1xuICAgIHRoaXMuaW5GbGlnaHQgPSBydW47XG4gICAgdHJ5IHtcbiAgICAgIGF3YWl0IHJ1bjtcbiAgICB9IGZpbmFsbHkge1xuICAgICAgaWYgKHRoaXMuaW5GbGlnaHQgPT09IHJ1bikgdGhpcy5pbkZsaWdodCA9IG51bGw7XG4gICAgfVxuICB9XG5cbiAgLyoqIFN0b3AgdGhlIHRpbWVyICh0ZXN0czsgcHJvY2VzcyB0ZWFyZG93bikuIEJ1ZmZlcmVkIHVuaXRzIHN0YXkgdW50aWwge0BsaW5rIGZsdXNofS4gKi9cbiAgc3RvcCgpOiB2b2lkIHtcbiAgICBpZiAodGhpcy50aW1lcikgY2xlYXJJbnRlcnZhbCh0aGlzLnRpbWVyKTtcbiAgICB0aGlzLnRpbWVyID0gbnVsbDtcbiAgfVxuXG4gIHByaXZhdGUgYXJtKCk6IHZvaWQge1xuICAgIGlmICh0aGlzLnRpbWVyKSByZXR1cm47XG4gICAgdGhpcy50aW1lciA9IHNldEludGVydmFsKCgpID0+IHsgdm9pZCB0aGlzLmZsdXNoKCk7IH0sIHRoaXMub3B0cy5mbHVzaE1zKTtcbiAgICB0aGlzLnRpbWVyLnVucmVmPy4oKTtcbiAgICBhcm1TaHV0ZG93bkZsdXNoKCk7XG4gIH1cblxuICBwcml2YXRlIGRpc3BhdGNoKG9yZ0lkOiBzdHJpbmcsIHF1b3RhVHlwZTogUXVvdGFUeXBlLCBncm91cHM6IE1ldGVyZWRJbmNyZW1lbnRHcm91cFtdKTogdm9pZCB7XG4gICAgdm9pZCB0aGlzLnNlbmRTYWZlbHkob3JnSWQsIHF1b3RhVHlwZSwgZ3JvdXBzKTtcbiAgfVxuXG4gIHByaXZhdGUgYXN5bmMgc2VuZFNhZmVseShvcmdJZDogc3RyaW5nLCBxdW90YVR5cGU6IFF1b3RhVHlwZSwgZ3JvdXBzOiBNZXRlcmVkSW5jcmVtZW50R3JvdXBbXSk6IFByb21pc2U8dm9pZD4ge1xuICAgIHRyeSB7XG4gICAgICBhd2FpdCB0aGlzLnNlbmQob3JnSWQsIHF1b3RhVHlwZSwgZ3JvdXBzKTtcbiAgICB9IGNhdGNoIChlcnIpIHtcbiAgICAgIGxvZ2dlci53YXJuKCdNZXRlcmVkIHF1b3RhIGluY3JlbWVudCBmYWlsZWQnLCB7IG9yZ0lkLCBxdW90YVR5cGUsIGVycm9yOiBTdHJpbmcoZXJyKSB9KTtcbiAgICAgIGVtaXRDb3VudGVyKCdxdW90YV9pbmNyZW1lbnRfZmFpbHVyZXNfdG90YWwnLCB7IHJlYXNvbjogJ2V4Y2VwdGlvbicsIHF1b3RhVHlwZSB9KTtcbiAgICB9XG4gIH1cbn1cblxuLyoqIE9uZSBiYXRjaGVyIHBlciBxdW90YSBjbGllbnQsIHNvIGEgdGVzdCdzIGZha2UgY2xpZW50IG5ldmVyIHNoYXJlcyBhIGJ1ZmZlci4gKi9cbmNvbnN0IGJhdGNoZXJzID0gbmV3IFdlYWtNYXA8UXVvdGFTZXJ2aWNlLCBRdW90YUluY3JlbWVudEJhdGNoZXI+KCk7XG5cbi8qKiBUaGUgbGl2ZSBiYXRjaGVycywgZm9yIHRoZSBzaHV0ZG93biBmbHVzaCAoYSBXZWFrTWFwIGNhbm5vdCBiZSBpdGVyYXRlZCkuICovXG5jb25zdCBsaXZlID0gbmV3IFNldDxRdW90YUluY3JlbWVudEJhdGNoZXI+KCk7XG5cbi8qKiBUaGUgYmF0Y2hlciBmZWVkaW5nIGBxdW90YVNlcnZpY2UuaW5jcmVtZW50TWV0ZXJlZGAuICovXG5leHBvcnQgZnVuY3Rpb24gYmF0Y2hlckZvcihxdW90YVNlcnZpY2U6IFF1b3RhU2VydmljZSwgc2VuZGVyOiBNZXRlcmVkSW5jcmVtZW50U2VuZGVyKTogUXVvdGFJbmNyZW1lbnRCYXRjaGVyIHtcbiAgbGV0IGIgPSBiYXRjaGVycy5nZXQocXVvdGFTZXJ2aWNlKTtcbiAgaWYgKCFiKSB7XG4gICAgYiA9IG5ldyBRdW90YUluY3JlbWVudEJhdGNoZXIoc2VuZGVyKTtcbiAgICBiYXRjaGVycy5zZXQocXVvdGFTZXJ2aWNlLCBiKTtcbiAgICBsaXZlLmFkZChiKTtcbiAgfVxuICByZXR1cm4gYjtcbn1cblxuLyoqXG4gKiBGbHVzaCBldmVyeSBwb2QtbG9jYWwgYmF0Y2hlci4gQ2FsbGVkIGJ5IGBydW5TZXJ2ZXJgIG9uY2UgdGhlIEhUVFAgc2VydmVyIGhhc1xuICogc3RvcHBlZCBhY2NlcHRpbmcgYW5kIGluLWZsaWdodCByZXF1ZXN0cyBoYXZlIGZpbmlzaGVkICh0aGVpciBgZmluaXNoYFxuICogbWV0ZXJzIGFyZSB0aGVuIGFsbCBidWZmZXJlZCkuIE5ldmVyIHJlamVjdHMuXG4gKi9cbmV4cG9ydCBhc3luYyBmdW5jdGlvbiBmbHVzaFF1b3RhSW5jcmVtZW50cygpOiBQcm9taXNlPHZvaWQ+IHtcbiAgYXdhaXQgUHJvbWlzZS5hbGwoWy4uLmxpdmVdLm1hcCgoYikgPT4gYi5mbHVzaCgpLmNhdGNoKCgpID0+IHVuZGVmaW5lZCkpKTtcbn1cblxubGV0IHNodXRkb3duQXJtZWQgPSBmYWxzZTtcblxuLyoqXG4gKiBTSUdURVJNIGZsdXNoIGZvciBwcm9jZXNzZXMgdGhhdCBkb24ndCBzaHV0IGRvd24gdGhyb3VnaCBgcnVuU2VydmVyYC5cbiAqXG4gKiBBcm1lZCBvbmx5IHdoZW4gdGhlIHByb2Nlc3MgQUxSRUFEWSBoYW5kbGVzIFNJR1RFUk0gKGV2ZXJ5IHNlcnZpY2UgcmVnaXN0ZXJzXG4gKiBpdHMgc2h1dGRvd24gYXQgc3RhcnR1cCwgYmVmb3JlIHRoZSBmaXJzdCByZXF1ZXN0IGlzIG1ldGVyZWQpOiBhZGRpbmcgdGhlXG4gKiBmaXJzdCBsaXN0ZW5lciB3b3VsZCBzd2l0Y2ggb2ZmIE5vZGUncyBkZWZhdWx0IGV4aXQtb24tU0lHVEVSTSBmb3IgYSBwcm9jZXNzXG4gKiB0aGF0IHJlbGllZCBvbiBpdCAoYSBDTEksIGEgdGVzdCB3b3JrZXIpLlxuICovXG5mdW5jdGlvbiBhcm1TaHV0ZG93bkZsdXNoKCk6IHZvaWQge1xuICBpZiAoc2h1dGRvd25Bcm1lZCB8fCBwcm9jZXNzLmxpc3RlbmVyQ291bnQoJ1NJR1RFUk0nKSA9PT0gMCkgcmV0dXJuO1xuICBzaHV0ZG93bkFybWVkID0gdHJ1ZTtcbiAgcHJvY2Vzcy5vbmNlKCdTSUdURVJNJywgKCkgPT4geyB2b2lkIGZsdXNoUXVvdGFJbmNyZW1lbnRzKCk7IH0pO1xufVxuIl19

package/lib/services/quota.d.ts

@@ -1,6 +1,107 @@
-import type { Response } from 'express';
-import type { QuotaType, QuotaCheckResult } from '../types/common.js';
+import type { Request, Response } from 'express';
+import { type MeteredIncrementGroup } from './quota-increment-batcher.js';
+import type { PrincipalType, QuotaType, QuotaCheckResult } from '../types/common.js';
+import { ErrorCode } from '../types/error-codes.js';
 import { type QuotaTier } from '../types/quota-tiers.js';
+/**
+ * Budget for one `check` of a COUNTER quota (`QUOTA_CHECK_TIMEOUT_MS`, default
+ * 300), with no retry.
+ *
+ * The check is a read gate that FAILS OPEN, run ahead of the request it gates:
+ * the old 5 s timeout × 2 attempts meant a slow quota service added up to 10 s
+ * to every metered request and then let it through anyway. A short budget turns
+ * the same incident into a ≤ 300 ms fail-open (counted on
+ * `quota_fail_open_total`, which alerts) — and with the per-pod check cache in
+ * front (api-server `checkQuota`) a healthy service is asked at most once per
+ * org/type per cache TTL.
+ *
+ * `storageBytes` keeps the client timeout and one retry: it is a MEASURED read
+ * (the quota service walks the org's registry namespace on a cold cache) behind
+ * the registry's fail-CLOSED push gate, where a premature timeout refuses a push.
+ */
+export declare const QUOTA_CHECK_TIMEOUT_MS: number;
+/** Who / what spent (or got back) a quota unit. All fields optional. */
+export interface QuotaUsageAttribution {
+    principalType?: PrincipalType;
+    /** The user id or service-account id behind the call (the token's `sub`). */
+    principalId?: string;
+    /** The revocable credential: the access key the token derives from, else its `jti`. */
+    keyId?: string;
+    /** `METHOD /route/:pattern` — the matched route PATTERN, never the raw URL (no ids, no query). */
+    route?: string;
+    /** The pipeline the call was made for (`:pipelineId` param or `X-Pipeline-Id` header). */
+    pipelineId?: string;
+    requestId?: string;
+    /**
+     * AI calls only: which product surface spent the unit — `ask`, `agent`,
+     * `agent-tool:<name>`, `pipeline-generate`, `plugin-generate`,
+     * `report-summary`. Set by api-server's `withAiCallReservation`.
+     */
+    feature?: string;
+    /** AI calls only: the model id the reservation was resolved to (never a key). */
+    model?: string;
+    /** AI calls only: whose key served the call — `platform`, `org` or `request` (BYOK). */
+    keySource?: AiKeySource;
+}
+/** Whose key serves an AI call (mirrors ai-core `KeySource`). Only `platform` spends `aiCalls`. */
+export type AiKeySource = 'platform' | 'org' | 'request';
+/** Token counts an AI call reports after it settles (see {@link QuotaService.reportUsage}). */
+export interface QuotaTokenUsage {
+    tokensIn: number;
+    tokensOut: number;
+}
+/**
+ * Attribution for the request being served: the verified principal, the matched
+ * route pattern, the pipeline (`:pipelineId` param or a sanitized
+ * `X-Pipeline-Id` header) and the request id. Read off `req` directly rather than
+ * the ambient audit context because the metering middleware runs in a `finish`
+ * listener, where the request's async scope is no longer guaranteed.
+ */
+export declare function quotaAttributionFromRequest(req: Request): QuotaUsageAttribution;
+/**
+ * The quota type a metered request actually spends.
+ *
+ * `apiCalls` is the PEOPLE's budget — an interactive session. A request made by
+ * a NON-INTERACTIVE credential spends `automationCalls` instead, never both, so
+ * a fleet of self-mutating pipelines can't exhaust the console for the humans
+ * in the same org:
+ *  - an org SERVICE ACCOUNT (`principalType: 'service_account'` —
+ *    platform-automation, pipeline-register, reporting-ingest, registry-push);
+ *  - a PERSONAL ACCESS KEY: the exchanged key token (`token_use: 'api_key'`)
+ *    and anything minted while holding one (`parentKeyId`). A PAT in a CI job
+ *    is automation whoever owns it — billing it as a person let one script
+ *    lock its owner's teammates out of the console. CI should still use a
+ *    service-account key (it survives the person leaving); either way the
+ *    traffic now lands on `automationCalls`.
+ * Every other type is returned unchanged: an AI call or a pipeline creation
+ * costs the same whoever makes it.
+ *
+ * `meterQuotaOnSuccess` and `checkQuota` call this themselves, so a service
+ * mounts both with `'apiCalls'` and the split happens per request.
+ */
+export declare function meteredQuotaTypeFor(req: Pick<Request, 'user'>, quotaType: QuotaType): QuotaType;
+/** Why a request is not metered/checked against apiCalls (see {@link callQuotaExemption}). */
+export type CallQuotaExemption = 'service' | 'impersonation' | 'on_behalf';
+/**
+ * Whether the apiCalls/automationCalls meter AND check skip this (verified)
+ * request, and why. One function for both halves, so the gate never refuses a
+ * call the meter would not bill:
+ *  - `service` — an internal service principal: peer traffic is caused by a
+ *    user action the originating service already metered;
+ *  - `impersonation` — a sysadmin impersonation session (`impersonatorId`).
+ *    Vendor staff looking at a tenant's console must neither spend the tenant's
+ *    budget nor be locked out by it while diagnosing an exhausted org; the
+ *    session is audited on its own trail;
+ *  - `on_behalf` — a user token FORWARDED by an internal service as part of a
+ *    request already metered where it arrived (the ask agent's tools,
+ *    pipeline's placeholder-plugin confirm), proven by a signed marker the
+ *    user cannot forge (middleware/on-behalf.ts).
+ * Applies to the CALL budgets only: reservation-gated quotas (pipelines,
+ * plugins …) are charged whoever asks. `aiCalls` mirrors the impersonation half
+ * on its own (api-server `withAiCallReservation` skips the slot when the
+ * session is an impersonation; the call is still audited).
+ */
+export declare function callQuotaExemption(req: Request): CallQuotaExemption | undefined;
 /**
  * Service-principal `Authorization` header for a quota call made on behalf of
  * `orgId` by the CURRENT service (`SERVICE_NAME`, the same identity the health
@@ -30,6 +131,16 @@ export interface QuotaReserveResult {
      * 503 instead of a misleading 429 "quota exceeded".
      */
     unavailable?: boolean;
+    /**
+     * Set when an `aiCalls` reservation was refused by one of the org's OPTIONAL
+     * AI controls rather than the period cap: {@link ErrorCode.AI_MEMBER_SHARE_EXCEEDED}
+     * (the caller's per-member share is spent) or
+     * {@link ErrorCode.AI_TOKEN_BUDGET_EXCEEDED} (the org's per-period token
+     * budget is spent). Paired with `exceeded: true` and the quota service's
+     * `message`; {@link sendQuotaReserveDenied} relays both.
+     */
+    code?: AiLimitCode;
+    message?: string;
     quota: {
         type: QuotaType;
         limit: number;
@@ -38,14 +149,32 @@ export interface QuotaReserveResult {
         resetAt?: string;
     };
 }
+/** The 429 codes an `aiCalls` reservation answers when an optional org AI control refuses it. */
+export type AiLimitCode = ErrorCode.AI_MEMBER_SHARE_EXCEEDED | ErrorCode.AI_TOKEN_BUDGET_EXCEEDED;
 /**
  * Quota service client interface.
  */
 export interface QuotaService {
     /** Check if quota is available (fail-open on error). */
     check(orgId: string, quotaType: QuotaType, authHeader: string, requestId?: string): Promise<QuotaCheckResult>;
-    /** Increment quota usage. Returns a promise so callers can optionally handle errors. */
-    increment(orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, requestId?: string): Promise<void>;
+    /**
+     * Increment quota usage. Returns a promise so callers can optionally handle
+     * errors. `attribution` defaults to the ambient request context.
+     */
+    increment(orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, attribution?: QuotaUsageAttribution): Promise<void>;
+    /**
+     * METERED increment of a pre-aggregated batch (the per-pod batcher's flush,
+     * see `quota-increment-batcher.ts`): the sum of `groups` is applied with
+     * PARTIAL admission — the counter moves by as much as fits under the cap and
+     * the remainder is recorded as `overshoot` — and each group's attribution is
+     * written to the usage ledger. Never a 429. Resolves with what was counted,
+     * or `null` when the quota service did not answer ok (counted on
+     * `quota_increment_failures_total`).
+     */
+    incrementMetered(orgId: string, quotaType: QuotaType, authHeader: string, groups: MeteredIncrementGroup[]): Promise<{
+        counted: number;
+        overshoot: number;
+    } | null>;
     /**
      * Atomic reserve  the same atomic check+increment as `increment`, but
      * parses the response so callers can see whether the slot was actually
@@ -56,7 +185,7 @@ export interface QuotaService {
      * unreachable, timeout, open circuit, non-ok, or a non-quota 429 — unless
      * `QUOTA_RESERVE_FAIL_OPEN=true` (see the policy note at the top of this file).
      */
-    reserve(orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, requestId?: string): Promise<QuotaReserveResult>;
+    reserve(orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, attribution?: QuotaUsageAttribution): Promise<QuotaReserveResult>;
     /**
      * Roll back a previously reserved slot. Fire-and-forget — never throws.
      * Pass `resetAtSnapshot` (the `quota.resetAt` observed when the slot was
@@ -64,7 +193,16 @@ export interface QuotaService {
      * between reserve and rollback, the server skips the decrement so it doesn't
      * steal capacity from the new period.
      */
-    decrement(orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, resetAtSnapshot?: string, requestId?: string): Promise<void>;
+    decrement(orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, resetAtSnapshot?: string, attribution?: QuotaUsageAttribution): Promise<void>;
+    /**
+     * Record what an AI call COST after it settled — its token counts — in the
+     * usage ledger, without moving any counter (a zero-unit ledger row carrying
+     * `tokensIn` / `tokensOut` under the call's attribution). Also how a call that
+     * spent NO `aiCalls` unit (a BYOK key, an impersonation session, a weight-0
+     * model) still shows up in the usage breakdown. Fire-and-forget — never throws.
+     * Optional so test doubles need not implement it.
+     */
+    reportUsage?(orgId: string, quotaType: QuotaType, authHeader: string, tokens: QuotaTokenUsage, attribution?: QuotaUsageAttribution): Promise<void>;
     /**
      * Get the org's quota `QuotaTier`. Used by the plugin-build queue partitioning
      * to route a build to the right per-tier queue. Fail-open returns `DEFAULT_TIER`
@@ -116,20 +254,26 @@ export interface QuotaServiceConfig {
  */
 export declare function createQuotaService(config?: QuotaServiceConfig): QuotaService;
 /**
- * Fire-and-forget quota METERING increment with standardized error logging.
+ * Fire-and-forget quota METERING increment.
+ *
+ * Buffered: the unit joins the pod's per-(org, type) aggregate and is sent with
+ * the next flush (`QUOTA_INCREMENT_FLUSH_MS`, see quota-increment-batcher.ts)
+ * as one metered increment — partial admission against the cap, the excess
+ * recorded as `overshoot`. The counter therefore lags by at most one flush
+ * interval. Only the request meter uses this; reservations stay synchronous.
  *
  * Authenticates as the calling SERVICE ({@link getQuotaServiceAuthHeader}), never
- * with the end user's token: `/quotas/:orgId/increment` is service-principal /
- * system-admin only, so a forwarded user JWT was always rejected (403) and the
- * metered quota (e.g. `apiCalls`, and the `api_pack` bundle raising it) never
- * moved.
+ * with the end user's token: `/quotas/:orgId/increment` is service-principal
+ * only, so a forwarded user JWT was always rejected (403) and the metered quota
+ * (e.g. `apiCalls`, and the `api_pack` bundle raising it) never moved.
  *
  * @param quotaService - Quota service client
  * @param orgId - Organization ID whose counter is incremented
  * @param quotaType - Quota type to increment
  * @param logWarn - Logging function for warnings
+ * @param attribution - Who spent the unit (defaults to the ambient request context)
  */
-export declare function incrementQuota(quotaService: QuotaService, orgId: string, quotaType: QuotaType, logWarn: (message: string, data?: unknown) => void): void;
+export declare function incrementQuota(quotaService: QuotaService, orgId: string, quotaType: QuotaType, logWarn: (message: string, data?: unknown) => void, attribution?: QuotaUsageAttribution): void;
 /**
  * Atomic reserve helper for the "reserve + commit / rollback" pattern.
  * Use for expensive resources (pipelines, plugins, AI calls) where the
@@ -151,7 +295,7 @@ export declare function incrementQuota(quotaService: QuotaService, orgId: string
  * }
  * ```
  */
-export declare function reserveQuota(quotaService: QuotaService, orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, requestId?: string): Promise<QuotaReserveResult>;
+export declare function reserveQuota(quotaService: QuotaService, orgId: string, quotaType: QuotaType, authHeader: string, amount?: number, attribution?: QuotaUsageAttribution): Promise<QuotaReserveResult>;
 /**
  * Answer a DENIED reservation (`reservation.exceeded === true`).
  *
@@ -166,4 +310,4 @@ export declare function sendQuotaReserveDenied(res: Response, quotaType: QuotaTy
  * Logs on failure but never throws  the action that needed the rollback
  * has already failed, no point compounding the error.
  */
-export declare function decrementQuota(quotaService: QuotaService, orgId: string, quotaType: QuotaType, authHeader: string, logWarn: (message: string, data?: unknown) => void, amount?: number, resetAtSnapshot?: string): void;
+export declare function decrementQuota(quotaService: QuotaService, orgId: string, quotaType: QuotaType, authHeader: string, logWarn: (message: string, data?: unknown) => void, amount?: number, resetAtSnapshot?: string, attribution?: QuotaUsageAttribution): void;