instar 1.3.783 → 1.3.785

package/dist/server/AgentServer.js

@@ -91,6 +91,14 @@ import { MeteredSpendGate } from '../core/MeteredSpendGate.js';
 import { RenderedPlanStore } from '../core/RenderedPlanStore.js';
 import { PinAttemptStore } from '../core/PinAttemptStore.js';
 import { SpendAlertResolver, buildStalePriceAlert } from '../core/SpendAlertResolver.js';
+import { SpendAlertDispatcher } from '../core/SpendAlertDispatcher.js';
+import { TelegramSpendTopicChannel, spendAlertDeliveryId } from '../core/TelegramSpendTopicChannel.js';
+import { SpendAlertEmitters } from '../core/SpendAlertEmitters.js';
+import { MachineIdentityManager } from '../core/MachineIdentity.js';
+import { PendingRelayStore as SpendAlertRelayStore } from '../messaging/pending-relay-store.js';
+import { ProviderCostReportStore } from '../monitoring/ProviderCostReportStore.js';
+import { ProviderReconciliationSweep } from '../monitoring/ProviderReconciliationSweep.js';
+import { meteredKeysFromChains } from '../core/routingSpendView.js';
 import { DEFAULT_FRESHNESS_SLA_DAYS } from '../core/routingPriceAuthority.js';
 import { METERED_ROUTING_DOORS } from '../data/llmBenchCoverage.js';
 import { DEFAULT_METERED_CAPS } from '../core/routingSpendView.js';
@@ -188,6 +196,14 @@ export class AgentServer {
     meteredSweepTimer = null;
     spendAlertResolver = null;
     spendAlertTimer = null;
+    /** Increment C alert layer (dryRun-first live-on-dev — FD-16). */
+    spendAlertDispatcher = null;
+    spendAlertEmitters = null;
+    /** Lazily-opened durable relay handle for money-critical alert delivery. */
+    spendAlertRelayStore = null;
+    /** Layer 1c provider-report store + reconciliation sweep (reporting-only; FD-21). */
+    providerCostReportStore = null;
+    reconSweepTimer = null;
     featureMetricsPruneTimer = null;
     a2aDeliveryTracker = null;
     tokenLedgerPoller = null;
@@ -654,6 +670,72 @@ export class AgentServer {
                     console.warn('[instar] pin-attempt store init failed (non-fatal — in-memory lockout only):', err);
                     this.pinAttemptStore = null;
                 }
+                // ── Layer 1c provider-report store + reconciliation sweep (FD-21 —
+                // REPORTING-only; rides the same dev-gated view flag as the summary;
+                // structurally excluded from the money gate, which imports nothing here).
+                if (routingSpendOn) {
+                    try {
+                        const rsCfg = options.config.routingSpend;
+                        this.providerCostReportStore = new ProviderCostReportStore({
+                            dbPath: path.join(options.config.stateDir, 'server-data', 'provider-cost-reports.db'),
+                            retentionDays: rsCfg?.providerReportRetentionDays ?? 400,
+                        });
+                        this.providerCostReportStore.prune(); // boot prune (batched — scal-F4)
+                        const doorToKey = new Map(meteredKeysFromChains().map((m) => [m.door, m.keyRef]));
+                        const sweep = new ProviderReconciliationSweep({
+                            store: this.providerCostReportStore,
+                            // Internal-derived per (keyRef, door): the metered doors' daily token
+                            // buckets × the as-of reviewed price (the same reporting math the
+                            // summary uses) — reporting-side reads only, never the money mutex.
+                            internalDerivedUsd: (sinceMs, untilMs) => {
+                                const ledger = this.featureMetricsLedger;
+                                const prices = this.routingPriceAuthority;
+                                if (!ledger || !prices)
+                                    return [];
+                                prices.reloadIfChanged();
+                                const sinceDays = Math.max(1, Math.ceil((untilMs - sinceMs) / 86_400_000) + 1);
+                                const buckets = ledger.spendTokenRollupDaily({ sinceDays });
+                                const agg = new Map();
+                                for (const b of buckets) {
+                                    const keyRef = doorToKey.get(b.door);
+                                    if (!keyRef)
+                                        continue; // metered doors only
+                                    if (b.bucketStartMs < sinceMs - 86_400_000 || b.bucketStartMs > untilMs)
+                                        continue;
+                                    const res = prices.resolve(b.door, b.modelId, b.bucketStartMs);
+                                    const cost = prices.reportingCost(res, b.tokensIn, b.tokensOut, b.tokensCached);
+                                    const e = agg.get(`${keyRef} ${b.door}`) ?? { keyRef, door: b.door, internalUsd: 0 };
+                                    e.internalUsd += cost.grossUsd;
+                                    agg.set(`${keyRef} ${b.door}`, e);
+                                }
+                                return [...agg.values()];
+                            },
+                            committedUsd: (keyRef) => {
+                                try {
+                                    return this.meteredSpendLedger?.committed(keyRef).committedLifetimeUsd ?? null;
+                                }
+                                catch {
+                                    // @silent-fallback-ok: committed is holder-known enrichment; a
+                                    // read failure records the comparison provider-vs-internal only.
+                                    return null;
+                                }
+                            },
+                            // The Increment-C emit surface (late-bound; dispatcher dryRun-soaked).
+                            onDrift: (keyRef, door, driftPct) => this.spendAlertEmitters?.onReconciliationDrift(keyRef, door, driftPct),
+                            driftAlertPct: rsCfg?.reconciliation?.driftAlertPct ?? 10,
+                        });
+                        const sweepMs = Math.max(1, rsCfg?.reconciliation?.sweepIntervalHours ?? 6) * 60 * 60 * 1000;
+                        this.reconSweepTimer = setInterval(() => void sweep.run(), sweepMs);
+                        this.reconSweepTimer.unref?.();
+                    }
+                    catch (err) {
+                        // @silent-fallback-ok: loud warn — a broken provider-report layer
+                        // degrades every row to internal-derived (labeled); it never touches
+                        // money admission or the router path.
+                        console.warn('[instar] provider-cost report layer init failed (non-fatal — internal-derived reporting remains):', err);
+                        this.providerCostReportStore = null;
+                    }
+                }
                 // ── Increment B MONEY layer (routing-control-room-spend §Layer 3 / Surface 2).
                 // EXPLICIT enable only (`routingSpend.money.enabled === true`) — a documented
                 // DARK_GATE_EXCLUSIONS action-bearing case (FD-16): NEVER resolveDevAgentGate.
@@ -679,6 +761,9 @@ export class AgentServer {
                                 prices: this.routingPriceAuthority,
                                 capsStore: this.routingSpendCapsStore,
                                 machineId,
+                                // Increment C observer (signal-only, late-bound — the alert layer
+                                // constructs after this block; null while dark).
+                                onGateEvent: (ev) => this.spendAlertEmitters?.onGateEvent(ev),
                                 // Single-machine trivially self-confirms (it IS the pool). On a
                                 // multi-machine pool the REAL designation re-confirmation plumbing
                                 // lands with the metered dispatch seam — until then the gate
@@ -700,51 +785,7 @@ export class AgentServer {
                         const alertsCfg = options.config.routingSpend;
                         const tg = this.telegramAdapter;
                         if (tg) {
-                            const persistPath = path.join(options.config.stateDir, 'state', 'routing-spend-alert-topic.json');
-                            this.spendAlertResolver = new SpendAlertResolver({
-                                configuredTopicId: () => {
-                                    const id = alertsCfg?.alerts?.telegramTopicId;
-                                    return typeof id === 'number' && Number.isFinite(id) ? id : undefined;
-                                },
-                                readPersistedTopicId: () => {
-                                    try {
-                                        const raw = JSON.parse(fs.readFileSync(persistPath, 'utf-8'));
-                                        return typeof raw.topicId === 'number' ? raw.topicId : undefined;
-                                    }
-                                    catch {
-                                        // @silent-fallback-ok: no persisted record yet — rung 2 simply misses.
-                                        return undefined;
-                                    }
-                                },
-                                persistTopicId: (topicId) => {
-                                    try {
-                                        const tmp = persistPath + '.tmp';
-                                        fs.writeFileSync(tmp, JSON.stringify({ topicId }), { mode: 0o600 });
-                                        fs.renameSync(tmp, persistPath);
-                                    }
-                                    catch {
-                                        // @silent-fallback-ok: a failed persist means the next resolve re-runs
-                                        // the ladder; creation stays fenced, so no duplicate can result.
-                                    }
-                                },
-                                // Single machine trivially self-confirms (it IS the serving lease); on a
-                                // multi-machine pool the confirmed-lease plumbing rides the C increment —
-                                // until then this machine does NOT create (fail toward the lifeline).
-                                servingLeaseConfirmedAgoMs: () => (poolOn ? null : 0),
-                                createTopic: async () => {
-                                    const t = await tg.createForumTopic('💰 Routing & Spend Alerts', undefined, {
-                                        origin: 'system',
-                                        bounded: true,
-                                        label: 'routing-spend-alerts',
-                                    });
-                                    return t.topicId;
-                                },
-                                sendToTopic: async (topicId, text) => {
-                                    await tg.sendToTopic(topicId, text);
-                                    return true;
-                                },
-                                lifelineTopicId: () => tg.getLifelineTopicId(),
-                            });
+                            this.spendAlertResolver ??= this.buildSpendAlertResolver(tg, options.config, poolOn);
                             const staleCheck = () => {
                                 try {
                                     const prices = this.routingPriceAuthority;
@@ -760,7 +801,15 @@ export class AgentServer {
                                         const res = prices.resolve(door, '__staleness-probe__', Date.now());
                                         if (res.priceStale && res.newestPointAgeDays !== null) {
                                             const meta = prices.doorMetaFor(door);
-                                            void resolver.emit(buildStalePriceAlert(door, res.newestPointAgeDays, meta?.freshnessSlaDays ?? DEFAULT_FRESHNESS_SLA_DAYS));
+                                            const alert = buildStalePriceAlert(door, res.newestPointAgeDays, meta?.freshnessSlaDays ?? DEFAULT_FRESHNESS_SLA_DAYS);
+                                            // Increment C: prefer the dispatcher (lane dedup + coalescing +
+                                            // dryRun soak); the B resolver-direct path remains the fallback
+                                            // when the alert layer is dark (byte-identical B behavior).
+                                            const dispatcher = this.spendAlertDispatcher;
+                                            if (dispatcher)
+                                                void dispatcher.dispatch(alert);
+                                            else
+                                                void resolver.emit(alert);
                                         }
                                     }
                                 }
@@ -786,6 +835,84 @@ export class AgentServer {
                         this.spendPlanStore = null;
                     }
                 }
+                // ── Increment C alert layer (routing-control-room-spend §Surface 2 Alerts).
+                // dryRun-FIRST live-on-dev (FD-16): `routingSpend.alerts.enabled` rides the
+                // dev-agent gate; `alerts.dryRun` defaults TRUE so a dev agent soaks the
+                // dispatch decisions in logs/routing-spend-alerts.jsonl before anything is
+                // delivered. Independent of the money layer — door-dark/fallback alerts can
+                // flow with no door ever armed (that is WHY the SERVING lease owns creation).
+                try {
+                    const alertsCfgC = options.config.routingSpend?.alerts;
+                    const alertsOn = resolveDevAgentGate(alertsCfgC?.enabled, options.config);
+                    const tgC = this.telegramAdapter;
+                    if (alertsOn && tgC) {
+                        const poolOnC = options.config.multiMachine?.sessionPool?.enabled === true;
+                        this.spendAlertResolver ??= this.buildSpendAlertResolver(tgC, options.config, poolOnC);
+                        const auditPath = path.join(options.config.projectDir, 'logs', 'routing-spend-alerts.jsonl');
+                        const auditSink = (entry) => {
+                            try {
+                                fs.mkdirSync(path.dirname(auditPath), { recursive: true });
+                                fs.appendFileSync(auditPath, JSON.stringify({ ts: new Date().toISOString(), ...entry }) + '\n', { mode: 0o600 });
+                            }
+                            catch {
+                                // @silent-fallback-ok: the audit line is observability — never blocks dispatch.
+                            }
+                        };
+                        const channel = new TelegramSpendTopicChannel({
+                            resolver: this.spendAlertResolver,
+                            sendToTopic: async (topicId, text) => {
+                                await tgC.sendToTopic(topicId, text);
+                                return true;
+                            },
+                            lifelineTopicId: () => tgC.getLifelineTopicId(),
+                            // Money-critical kinds ride the durable relay when the store opens
+                            // (retry-until-delivered — delivery-robustness Layers 2/3; the
+                            // DeliveryFailureSentinel drains this store). Opened lazily,
+                            // per-agent-id isolated, degrade-to-direct-send on ANY failure.
+                            enqueueDurable: (topicId, text) => {
+                                try {
+                                    this.spendAlertRelayStore ??= SpendAlertRelayStore.open(options.config.projectName, options.config.stateDir);
+                                    const store = this.spendAlertRelayStore;
+                                    const id = spendAlertDeliveryId(topicId, text);
+                                    return store.enqueue({
+                                        delivery_id: id,
+                                        topic_id: topicId,
+                                        text_hash: id,
+                                        text,
+                                        http_code: 0,
+                                        attempted_port: options.config.port ?? 0,
+                                        next_attempt_at: new Date().toISOString(),
+                                    });
+                                }
+                                catch {
+                                    // @silent-fallback-ok: no sqlite / store unavailable — the channel
+                                    // degrades to the direct send + lifeline ladder (delivery degrades,
+                                    // never dies; the dispatcher's latch still requires a confirmed leg).
+                                    return false;
+                                }
+                            },
+                            audit: auditSink,
+                        });
+                        this.spendAlertDispatcher = new SpendAlertDispatcher({
+                            channels: [channel],
+                            dryRun: alertsCfgC?.dryRun !== false, // FD-16 dryRun-first
+                            auditPath,
+                        });
+                        const machineIdC = options.config.machineId ?? 'single-machine';
+                        this.spendAlertEmitters = new SpendAlertEmitters({
+                            dispatcher: this.spendAlertDispatcher,
+                            machineId: machineIdC,
+                        });
+                    }
+                }
+                catch (err) {
+                    // @silent-fallback-ok: loud warn — a broken alert layer degrades to the
+                    // Increment-B behavior (resolver-direct stale-price emission); it never
+                    // touches money admission or the router path.
+                    console.warn('[instar] routing-spend alert layer init failed (non-fatal — B-level alerts remain):', err);
+                    this.spendAlertDispatcher = null;
+                    this.spendAlertEmitters = null;
+                }
                 // Observable Intelligence × Responsible Resource: the audit trail is kept
                 // long enough to see behaviour/performance trends, then aged out — never
                 // hoarded forever. Default 30d; tune via monitoring.featureMetrics.retentionDays
@@ -2126,6 +2253,7 @@ export class AgentServer {
             meteredSpendGate: this.meteredSpendGate,
             spendPlanStore: this.spendPlanStore,
             pinAttemptStore: this.pinAttemptStore,
+            providerCostReportStore: this.providerCostReportStore,
             resourceLedger: this.resourceLedger,
             processFootprintMonitor: this.processFootprintMonitor,
             approvalLedger: this.approvalLedger,
@@ -3917,6 +4045,15 @@ export class AgentServer {
                     clearInterval(this.spendAlertTimer);
             }
             catch { /* @silent-fallback-ok: timer teardown is best-effort cleanup at shutdown */ }
+            try {
+                await this.spendAlertDispatcher?.flushDigest();
+            }
+            catch { /* @silent-fallback-ok: a failed final digest flush loses only a coalesced NOTICE at shutdown */ }
+            try {
+                if (this.reconSweepTimer)
+                    clearInterval(this.reconSweepTimer);
+            }
+            catch { /* @silent-fallback-ok: timer teardown is best-effort cleanup at shutdown */ }
             this.featureMetricsPruneTimer = null;
         }
         if (this.tokenLedger) {
@@ -4017,6 +4154,74 @@ export class AgentServer {
      * its map in memory, so constructing a second instance on the same file would
      * lose updates between the two caches.
      */
+    /** Increment C — the router fan-out + gate observer reach the emitters here (late-bound; null while dark). */
+    getSpendAlertEmitters() {
+        return this.spendAlertEmitters;
+    }
+    /**
+     * Shared builder for the "💰 Routing & Spend Alerts" topic resolver — used by
+     * BOTH the Increment-B money block (stale-price cadence) and the Increment-C
+     * alert layer, whichever constructs first. Rung 2 carries both halves: the
+     * machine-local persisted record AND the pool-published registry field
+     * (FD-6 — a future serving-lease holder inherits the id, never re-creates).
+     */
+    buildSpendAlertResolver(tg, config, poolOn) {
+        const persistPath = path.join(config.stateDir, 'state', 'routing-spend-alert-topic.json');
+        const machineId = config.machineId;
+        const registry = new MachineIdentityManager(config.stateDir);
+        return new SpendAlertResolver({
+            configuredTopicId: () => {
+                const id = config.routingSpend?.alerts?.telegramTopicId;
+                return typeof id === 'number' && Number.isFinite(id) ? id : undefined;
+            },
+            readPersistedTopicId: () => {
+                try {
+                    const raw = JSON.parse(fs.readFileSync(persistPath, 'utf-8'));
+                    return typeof raw.topicId === 'number' ? raw.topicId : undefined;
+                }
+                catch {
+                    // @silent-fallback-ok: no persisted record yet — rung 2 simply misses.
+                    return undefined;
+                }
+            },
+            persistTopicId: (topicId) => {
+                try {
+                    const tmp = persistPath + '.tmp';
+                    fs.writeFileSync(tmp, JSON.stringify({ topicId }), { mode: 0o600 });
+                    fs.renameSync(tmp, persistPath);
+                }
+                catch {
+                    // @silent-fallback-ok: a failed persist means the next resolve re-runs
+                    // the ladder; creation stays fenced, so no duplicate can result.
+                }
+            },
+            // FD-6 rung 2, pool half: the replicated machine-registry field.
+            readPoolPublishedTopicId: () => registry.readAnyRoutingSpendAlertTopic(),
+            publishTopicId: (topicId) => {
+                if (!machineId)
+                    return; // single-machine installs: the local record suffices
+                registry.updateRoutingSpendAlertTopic(machineId, topicId);
+            },
+            // Single machine trivially self-confirms (it IS the serving lease); on a
+            // multi-machine pool the confirmed-lease plumbing is a tracked follow-up —
+            // until then this machine does NOT create (fail toward the lifeline)
+            // <!-- tracked: CMT-1929 -->.
+            servingLeaseConfirmedAgoMs: () => (poolOn ? null : 0),
+            createTopic: async () => {
+                const t = await tg.createForumTopic('💰 Routing & Spend Alerts', undefined, {
+                    origin: 'system',
+                    bounded: true,
+                    label: 'routing-spend-alerts',
+                });
+                return t.topicId;
+            },
+            sendToTopic: async (topicId, text) => {
+                await tg.sendToTopic(topicId, text);
+                return true;
+            },
+            lifelineTopicId: () => tg.getLifelineTopicId(),
+        });
+    }
     getTopicOperatorStore() {
         return this.topicOperatorStore;
     }