instar 1.3.1339 → 1.3.1341

package/dist/commands/server.js

@@ -18060,17 +18060,57 @@ export async function startServer(options) {
         // backoff (body recovered from the canonical outbox), and after the attempt
         // cap raises ONE aggregated attention item per dark peer. Recording/sending
         // only — no blocking authority. Ships OFF (it re-sends + escalates).
+        // REWORKED (a2a-single-agent-identity §3): constructed when EITHER
+        // `monitoring.a2aRedelivery.enabled` OR the resolved `threadline.peerDarkNotice`
+        // gate is on; `redeliver` only under the former (escalate-only otherwise).
+        // Under the peer-dark gate the operator notice is per PEER with the
+        // deterministic id `a2a-peer-dark:<agent>:<peerFp>`, raised by the AWAKE
+        // machine after it healed its own side, resolved from pool-scope evidence,
+        // 12 h cooldown; a local relay outage collapses to ONE aggregated item.
+        // Dry-run (default) logs would-raise rows to logs/a2a-peer-dark.jsonl.
         try {
             const a2aDelivCfg = config.monitoring?.a2aRedelivery ?? {};
-            if (a2aDelivCfg.enabled && a2aDeliveryTracker) {
+            const { resolvePeerDarkNoticeConfig, createPeerDarkAuditWriter, resolvePeerDarkAuditPath } = await import('../threadline/peerDark.js');
+            const peerDarkCfg = resolvePeerDarkNoticeConfig(config);
+            if ((a2aDelivCfg.enabled || peerDarkCfg.enabled) && a2aDeliveryTracker) {
                 const { A2ARedeliverySentinel, DEFAULT_A2A_REDELIVERY_CONFIG } = await import('../monitoring/A2ARedeliverySentinel.js');
+                const priorityMap = {
+                    high: 'HIGH', medium: 'NORMAL', low: 'LOW',
+                };
+                // Pool-scope read of one peer's health on my OTHER machines (raise AND
+                // resolve both read it — a peer that answered on another machine is not
+                // dark). Bounded per peer; a dark/slow peer machine is simply absent.
+                const poolPeerHealth = async (peerFp) => {
+                    const urls = _resolvePeerUrls?.() ?? [];
+                    if (urls.length === 0)
+                        return [];
+                    const rows = await Promise.allSettled(urls.map(async ({ machineId, url }) => {
+                        const ac = new AbortController();
+                        const t = setTimeout(() => ac.abort(), 5000);
+                        try {
+                            const r = await fetch(`${url}/threadline/peers/${encodeURIComponent(peerFp)}/health`, {
+                                headers: { Authorization: `Bearer ${config.authToken ?? ''}`, 'X-Instar-AgentId': config.projectName ?? '' },
+                                signal: ac.signal,
+                            });
+                            if (!r.ok)
+                                throw new Error(`HTTP ${r.status}`);
+                            const b = await r.json();
+                            return { machineId, lastAckedAt: b.lastAckedAt ?? null, lastInboundAt: b.lastInboundAt ?? null, lastDeliveredAt: b.lastDeliveredAt ?? null };
+                        }
+                        finally {
+                            clearTimeout(t);
+                        }
+                    }));
+                    return rows.flatMap((r) => (r.status === 'fulfilled' ? [r.value] : []));
+                };
                 const a2aRedeliverySentinel = new A2ARedeliverySentinel({
                     tracker: a2aDeliveryTracker,
+                    agentId: config.projectName,
                     // Re-send: recover the body from the canonical outbox by messageId,
                     // then re-emit via the relay. Body missing → return false (the
                     // sentinel leaves it awaiting-ack and escalates at the cap, never
-                    // fabricating a send).
-                    redeliver: (threadlineRelayClient && listenerManager)
+                    // fabricating a send). Only under the legacy gate.
+                    redeliver: (a2aDelivCfg.enabled && threadlineRelayClient && listenerManager)
                         ? (entry) => {
                             const stored = listenerManager.readCanonicalOutboxEntry(entry.messageId);
                             if (!stored)
@@ -18081,27 +18121,93 @@ export async function startServer(options) {
                         : undefined,
                     raiseAttention: telegram
                         ? async (item) => {
-                            const priorityMap = {
-                                high: 'HIGH', medium: 'NORMAL', low: 'LOW',
-                            };
-                            return telegram.createAttentionItem({
-                                id: `a2a-redelivery-${Date.now()}`,
+                            // Legacy per-message escalation keeps its stamped id; the §3
+                            // items carry a deterministic id and REOPEN on a later episode.
+                            if (!item.id) {
+                                return telegram.createAttentionItem({
+                                    id: `a2a-redelivery-${Date.now()}`,
+                                    title: item.title,
+                                    summary: item.body.slice(0, 160),
+                                    description: item.body,
+                                    category: 'a2a-redelivery',
+                                    priority: priorityMap[item.priority ?? 'medium'] ?? 'NORMAL',
+                                    sourceContext: item.source ?? 'a2a-redelivery',
+                                });
+                            }
+                            return telegram.upsertAttentionItem({
+                                id: item.id,
                                 title: item.title,
                                 summary: item.body.slice(0, 160),
                                 description: item.body,
-                                category: 'a2a-redelivery',
+                                category: 'a2a-peer-dark',
                                 priority: priorityMap[item.priority ?? 'medium'] ?? 'NORMAL',
-                                sourceContext: item.source ?? 'a2a-redelivery',
+                                sourceContext: item.source ?? 'a2a-peer-dark',
+                            });
+                        }
+                        : undefined,
+                    // Resolve: the resolve line lands on the hub through the same upsert
+                    // (body changed → re-posted), then the item is closed silently.
+                    resolveAttention: telegram
+                        ? async (id, line) => {
+                            const existing = telegram.getAttentionItem(id);
+                            if (!existing)
+                                return false;
+                            await telegram.upsertAttentionItem({
+                                id,
+                                title: existing.title,
+                                summary: line.slice(0, 160),
+                                description: line,
+                                category: existing.category,
+                                priority: existing.priority,
+                                sourceContext: existing.sourceContext,
                             });
+                            return telegram.updateAttentionStatus(id, 'DONE', { silent: true });
+                        }
+                        : undefined,
+                    attentionState: telegram
+                        ? (id) => {
+                            const it = telegram.getAttentionItem(id);
+                            return it ? { status: it.status, updatedAt: it.updatedAt, createdAt: it.createdAt } : null;
                         }
                         : undefined,
-                }, { ...mergeDefaults(DEFAULT_A2A_REDELIVERY_CONFIG, a2aDelivCfg), enabled: true });
+                    relayState: () => {
+                        const rc = threadlineRelayClient;
+                        if (!rc)
+                            return 'not-configured';
+                        if (rc.connectionState === 'connected')
+                            return 'connected';
+                        return threadlineGetLastRelayEvent?.()?.terminal === true ? 'displaced' : 'disconnected';
+                    },
+                    // §3.3: a standby's relay is disconnected BY DESIGN; only the awake
+                    // (telegram-polling) machine may heal or raise. The boot-time standby
+                    // fact is the same one the relay-forward route reads.
+                    isAwake: () => !(_a2aRelayForwardCtx?.relaySuppressedByStandby === true),
+                    reconnectRelay: threadlineRelayClient ? () => threadlineRelayClient.reconnectRelay() : undefined,
+                    refreshPresence: threadlineRelayClient ? () => threadlineRelayClient.refreshPresence() : undefined,
+                    peerConnectedNow: threadlineRelayClient ? (fp) => threadlineRelayClient.peerConnectedNow(fp) : undefined,
+                    peerPresence: threadlineRelayClient ? (fp) => threadlineRelayClient.peerPresence(fp) : undefined,
+                    // identitySelfCheck (§2) is wired by the identity-coherence check once
+                    // it lands; absent, the heal records `unknown` and never supersedes.
+                    poolPeerHealth,
+                    audit: createPeerDarkAuditWriter(resolvePeerDarkAuditPath(config.stateDir), (l) => console.warn(l)),
+                }, {
+                    ...mergeDefaults(DEFAULT_A2A_REDELIVERY_CONFIG, a2aDelivCfg),
+                    enabled: a2aDelivCfg.enabled === true,
+                    peerDark: {
+                        enabled: peerDarkCfg.enabled,
+                        dryRun: peerDarkCfg.dryRun,
+                        queuedDarkAfterMs: peerDarkCfg.queuedDarkAfterMs,
+                        perPeer: peerDarkCfg.perPeer,
+                        cooldownMs: peerDarkCfg.cooldownMs,
+                    },
+                });
                 a2aRedeliverySentinel.start();
                 globalThis.__instarA2ARedeliverySentinel = a2aRedeliverySentinel;
-                console.log(pc.green('  A2ARedeliverySentinel: armed (A2A redelivery + dark-peer escalation)'));
+                console.log(pc.green(`  A2ARedeliverySentinel: armed (redelivery ${a2aDelivCfg.enabled ? 'on' : 'off'}; ` +
+                    `peer-dark notice ${peerDarkCfg.enabled ? (peerDarkCfg.dryRun ? 'dry-run' : 'LIVE') : 'off'})`));
             }
             else {
-                console.log(pc.dim('  A2ARedeliverySentinel: disabled (monitoring.a2aRedelivery.enabled=false; the ship-OFF default)'));
+                console.log(pc.dim('  A2ARedeliverySentinel: disabled (monitoring.a2aRedelivery.enabled=false and threadline.peerDarkNotice dark; the ship-OFF default)'));
             }
         }
         catch (err) {
@@ -25216,15 +25322,217 @@ export async function startServer(options) {
                             const v = config.multiMachine?.relayTimeoutMs;
                             return typeof v === 'number' && v > 0 ? v : 15_000;
                         })();
+                        // ── a2a-single-agent-identity §4 (ACT-058): forward-to-holder ──
+                        // A token-holding machine that does NOT hold the serving lease (a
+                        // pool standby that legitimately owns a topic) forwards its reply to
+                        // the lease holder instead of holding it silently; a failed forward
+                        // becomes a DURABLE, reported hold. The forward itself is switchable
+                        // (`messaging[].config.messageOrigin.forwardToHolder.enabled`); the
+                        // durable held state, its reason and the notice are not.
+                        const originForwardMod = await import('../messaging/telegram-origin/OriginForwardToHolder.js');
+                        const originConfigMod = await import('../messaging/telegram-origin/OriginConfig.js');
+                        const hasUsableBotToken = typeof originBotToken === 'string' && originBotToken.length > 0;
+                        const forwardSwitchOn = () => {
+                            try {
+                                const messaging = liveConfig.get('messaging', config.messaging);
+                                const entry = Array.isArray(messaging) ? messaging.find(m => m?.type === 'telegram') : undefined;
+                                return originConfigMod.originForwardToHolderEnabled(entry?.config?.messageOrigin);
+                            }
+                            catch {
+                                return true;
+                            } // @silent-fallback-ok — an unreadable switch keeps the reachability floor on
+                        };
+                        const forwardLease = {
+                            selfMachineId: meshSelfId ?? '',
+                            leaseHolder: () => coordinator.getSyncStatus().leaseHolder ?? null,
+                            holdsLease: () => coordinator.holdsLease(),
+                            isHolderHealthy: (machineId) => { try {
+                                return leaseCoordinatorRef ? leaseCoordinatorRef.isHolderHealthy(machineId) : false;
+                            }
+                            catch {
+                                return false;
+                            } }, // @silent-fallback-ok — an unreadable lease reads as NOT healthy: settling, never a blind forward
+                        };
+                        const holderNickname = (machineId) => {
+                            try {
+                                return coordinator.managers.identityManager.loadRegistry().machines[machineId]?.nickname ?? `machine ${machineId.slice(0, 8)}`;
+                            }
+                            catch {
+                                return `machine ${machineId.slice(0, 8)}`;
+                            }
+                        };
+                        const heldAuditPath = path.join(config.stateDir, 'logs', 'telegram-origin-held.jsonl');
+                        const heldAudit = (row) => {
+                            try {
+                                fs.mkdirSync(path.dirname(heldAuditPath), { recursive: true });
+                                fs.appendFileSync(heldAuditPath, JSON.stringify({ machineId: meshSelfId, ...row }) + '\n');
+                            }
+                            catch { /* @silent-fallback-ok — audit is observability only */ }
+                        };
+                        const heldCollapser = new originForwardMod.HeldForwardItemCollapser({ now: Date.now });
+                        const raiseHeldItem = async (input) => {
+                            const verdict = heldCollapser.record(input.topicId);
+                            const holderName = input.holder ? holderNickname(input.holder) : 'no machine currently holds the serving lease';
+                            try {
+                                await telegram.createAttentionItem({ id: verdict.itemId, category: 'telegram-origin', priority: 'HIGH',
+                                    title: verdict.aggregate ? `Replies held on ${holderNickname(meshSelfId ?? '')}: ${heldCollapser.topics().length} topics` : `Reply to topic ${input.topicId} is held`,
+                                    summary: verdict.aggregate
+                                        ? `Replies for topics ${heldCollapser.topics().join(', ')} could not be forwarded to the lease holder (${holderName}). They are held durably and retried; see GET /telegram/origins/status → held.`
+                                        : `My reply to topic ${input.topicId} could not be forwarded to the lease holder (${holderName}): ${input.reason}. It is held durably (hold_reason: lease-not-held) and retried on the recovery schedule.`,
+                                    sourceContext: 'telegram-origin-forward-to-holder' }, { hubOnly: true });
+                            }
+                            catch (error) {
+                                heldAudit({ phase: 'item-failed', topicId: input.topicId, error: error instanceof Error ? error.message : String(error) });
+                            }
+                        };
+                        const tgConfig = () => originTelegramConfig.config;
+                        const sendToHolder = (holder, timeoutMs = Math.min(relayTimeoutMs, originForwardMod.FORWARD_RPC_TIMEOUT_MS)) => {
+                            const url = peerUrl(holder);
+                            return (command) => {
+                                if (!url)
+                                    return Promise.reject(new Error('origin-peer-unreachable'));
+                                // §4.2 — `capabilities` + `submit` share the 30 s attempt budget.
+                                return meshClient.send({ machineId: holder, url }, command, 0, { timeoutMs });
+                            };
+                        };
+                        const forwardDeps = (runtime) => ({
+                            lease: forwardLease,
+                            prepare: async (holder, input, operationId) => {
+                                const service = runtime.service;
+                                const { applyTelegramFormatter } = await import('../messaging/TelegramAdapter.js');
+                                const params = applyTelegramFormatter('sendMessage', { chat_id: input.chatId, text: input.text, parse_mode: 'Markdown',
+                                    ...(input.topicId > 1 ? { message_thread_id: input.topicId } : {}), ...(input.silent ? { disable_notification: true } : {}) }, input.formatMode ?? tgConfig().formatMode).outgoingParams;
+                                const accountId = String(originBotToken).split(':')[0];
+                                const prepare = () => service.prepareBot({ method: 'sendMessage', accountId, params, executionOwnerMachineId: holder,
+                                    ...(operationId === undefined ? {} : { operationId }), policyInput: service.currentSendPolicyInput(input.text, input.kindMetadata) });
+                                const operation = service.hasProducerContext() ? prepare() : service.runAsUnboundAutomation('telegram-server', prepare);
+                                service.authorizeSendPolicyDispatch(operation.record);
+                                await service.recordIntent(operation);
+                                return operation;
+                            },
+                            submit: (holder, operation) => originForwardMod.submitOriginToHolder({ operation, send: sendToHolder(holder), expectedOwner: holder }),
+                            hold: async (operation, detail) => {
+                                // Admit locally (the row + sealed payload) then name the reason
+                                // through the EXISTING operation-state writer; the recovery tick
+                                // re-forwards this same operation id.
+                                await runtime.service.admit(operation);
+                                const recorded = await runtime.store.recordOperationState({ operationId: operation.record.operationId, state: 'held', holdReason: originForwardMod.HOLD_REASON_LEASE_NOT_HELD, holdDetail: detail });
+                                await runtime.refreshHeldForward({ force: true });
+                                return recorded;
+                            },
+                            notify: async (holder, topicId) => {
+                                const text = originForwardMod.heldForwardNoticeText(holderNickname(holder));
+                                const { relayOriginBot } = await import('../messaging/telegram-origin/OriginMeshRelay.js');
+                                // Three short RPCs (5 s each): the notice must never push the reply
+                                // request past its route timeout — worst case settle 15 s + attempt 30 s + notice 15 s.
+                                const r = await runtime.service.runAsUnboundAutomation('telegram-server', () => relayOriginBot({ runtime, topicId, text, chatId: tgConfig().chatId,
+                                    formatMode: tgConfig().formatMode, send: sendToHolder(holder, 5_000), executionOwnerMachineId: holder }));
+                                return Number.isSafeInteger(r.messageId);
+                            },
+                            audit: heldAudit, sleep: ms => new Promise(resolve => setTimeout(resolve, ms)), now: Date.now,
+                        });
+                        if (telegramOriginBoot && hasUsableBotToken && meshSelfId) {
+                            const runtime = telegramOriginBoot.runtime;
+                            telegram.holderForwardPolicy = {
+                                decide: () => {
+                                    if (!forwardSwitchOn() || _sessionPoolStage() === 'dark')
+                                        return 'local';
+                                    const route = originForwardMod.classifyLeaseRoute(forwardLease);
+                                    return route.kind === 'self' ? 'local' : route.kind === 'peer' ? 'forward' : 'settling';
+                                },
+                            };
+                            // §4.2 — recovery re-forwards the SAME operation; an outcome-unknown
+                            // attempt is resolved at the old holder's receipt, never re-sent blind.
+                            runtime.options.forwardRecovery = (operation, row) => originForwardMod.recoverForwardedHold({
+                                lease: forwardLease, prepare: forwardDeps(runtime).prepare, submit: forwardDeps(runtime).submit, audit: heldAudit, now: Date.now,
+                                receipt: async (machineId, operationId) => {
+                                    try {
+                                        const reply = await sendToHolder(machineId)({ type: 'telegram-origin', protocol: 'instar-telegram-origin-v1', action: 'receipt', operationId });
+                                        const result = reply.result;
+                                        if (reply.ok && result?.ok && result.operationId === operationId) {
+                                            const state = String(result.state);
+                                            if (state === 'accepted' && result.originReceiptConfirmed === true)
+                                                return { state: 'accepted', receiptJson: JSON.stringify({ confirmedVia: 'receipt', deliveryMachineId: machineId, operationId }) };
+                                            // Definite non-admission: never seen, evidence only, or terminal without delivery.
+                                            if (['no-record', 'not-admitted', 'known-failed', 'expired', 'suppressed', 'superseded'].includes(state))
+                                                return { state: 'not-accepted' };
+                                            // The holder has custody (its own recovery delivers it): wait, never re-forward.
+                                            if (['admitted', 'held', 'partial', 'accepted'].includes(state))
+                                                return { state: 'owned-by-holder' };
+                                            return { state: 'unreachable' };
+                                        }
+                                        // `origin-receipt-unavailable` / any refusal is an UNKNOWN (a store outage looks the same as absence to an old holder): keep waiting.
+                                        return { state: 'unreachable' };
+                                    }
+                                    catch {
+                                        return { state: 'unreachable' };
+                                    }
+                                },
+                                resolve: (operationId, deliveryMachineId, receiptJson) => runtime.store.recordForwardedAcceptance({ operationId, deliveryMachineId, receiptJson }),
+                                rehold: async (operationId, detail) => { await runtime.store.recordOperationState({ operationId, state: 'held', holdReason: originForwardMod.HOLD_REASON_LEASE_NOT_HELD, holdDetail: detail }); },
+                                supersede: (operationId, next, detail) => runtime.store.supersedeWithAdmission({ operationId, admission: next.admission,
+                                    holdReason: originForwardMod.HOLD_REASON_LEASE_NOT_HELD, holdDetail: detail }),
+                                sendLocal: async (detail) => {
+                                    const sent = await telegram.sendToTopic(detail.topicId, detail.text, { silent: detail.silent, kindMetadata: detail.kindMetadata, provenance: 'automation' });
+                                    return { messageId: sent.messageId };
+                                },
+                            }, row, operation);
+                            // §4.2 — a DIRECT-path hold (`authorize` refused while not holding the lease:
+                            // pool dark, or the lease moved between the adapter decision and authorize).
+                            // Its sealed record is owned by THIS machine, so it cannot be forwarded; it is
+                            // re-admitted and replayed the moment the lease returns (runtime.recoverHeld).
+                            // Report it exactly like a failed forward: the fixed notice through the
+                            // holder when one is reachable, otherwise the collapsed attention item.
+                            runtime.options.onLeaseHoldObserved = async (row) => {
+                                const topicId = Number(row.destination.topicId);
+                                heldAudit({ phase: 'direct-hold-observed', topicId: row.destination.topicId, operationId: row.operationId });
+                                if (!Number.isSafeInteger(topicId))
+                                    return;
+                                const route = originForwardMod.classifyLeaseRoute(forwardLease);
+                                let noticeDelivered = false;
+                                if (route.kind === 'peer' && forwardSwitchOn()) {
+                                    try {
+                                        noticeDelivered = await forwardDeps(runtime).notify(route.holder, topicId);
+                                    }
+                                    catch {
+                                        noticeDelivered = false;
+                                    } // @silent-fallback-ok — an undelivered notice falls through to the attention item below
+                                }
+                                heldAudit({ phase: noticeDelivered ? 'notice-sent' : 'notice-not-sent', topicId, operationId: row.operationId, holder: route.kind === 'peer' ? route.holder : null });
+                                if (!noticeDelivered)
+                                    await raiseHeldItem({ topicId, holder: route.kind === 'peer' ? route.holder : null, operationId: row.operationId, lastAttempt: 'refused', reason: 'lease-not-held (direct path)' });
+                            };
+                            runtime.options.onHeldForwardExpired = async (row) => {
+                                const topicId = Number(row.destination.topicId);
+                                heldAudit({ phase: 'expired', topicId: row.destination.topicId, operationId: row.operationId, lastAttempt: row.holdDetail?.lastAttempt ?? null });
+                                if (Number.isSafeInteger(topicId))
+                                    heldCollapser.clear(topicId);
+                                await telegram.createAttentionItem({ id: `telegram-origin-held:${row.destination.topicId ?? row.operationId}:expired`, category: 'telegram-origin', priority: 'HIGH',
+                                    title: `Held reply to topic ${row.destination.topicId ?? '?'} expired`, summary: originForwardMod.expiredForwardWording(row),
+                                    sourceContext: 'telegram-origin-forward-to-holder' }, { hubOnly: true });
+                            };
+                        }
                         telegram.outboundRelay = async (topicId, text, opts) => {
                             if (telegramOriginBoot) {
+                                const { TelegramOriginHoldError, OriginForwardSettledLocallyError } = await import('../messaging/telegram-origin/types.js');
+                                const tg = tgConfig();
+                                if (hasUsableBotToken && meshSelfId) {
+                                    // §4.1/§4.2 — settle, forward to the holder, ladder, then hold durably.
+                                    const runtime = telegramOriginBoot.runtime;
+                                    const outcome = await originForwardMod.forwardReplyToHolder(forwardDeps(runtime), { topicId, chatId: tg.chatId, text, silent: opts?.silent, formatMode: tg.formatMode, kindMetadata: opts?.kindMetadata });
+                                    if (outcome.kind === 'local')
+                                        throw new OriginForwardSettledLocallyError();
+                                    if (outcome.kind === 'sent')
+                                        return { messageId: outcome.messageId, topicId, destinationStoreConfirmed: true };
+                                    if (!outcome.noticeDelivered)
+                                        void raiseHeldItem({ topicId, holder: outcome.holder, operationId: outcome.operationId, lastAttempt: outcome.lastAttempt, reason: outcome.reason });
+                                    throw new TelegramOriginHoldError(originForwardMod.HOLD_REASON_LEASE_NOT_HELD, outcome.operationId, 'held');
+                                }
                                 const holder = coordinator.getSyncStatus().leaseHolder;
                                 const url = holder ? peerUrl(holder) : null;
-                                const { TelegramOriginHoldError } = await import('../messaging/telegram-origin/types.js');
                                 if (!holder || !url || holder === meshSelfId)
                                     throw new TelegramOriginHoldError('origin-credential-owner-unavailable');
                                 const { relayOriginBot } = await import('../messaging/telegram-origin/OriginMeshRelay.js');
-                                const tg = originTelegramConfig.config;
                                 return relayOriginBot({ runtime: telegramOriginBoot.runtime, topicId, text, chatId: tg.chatId,
                                     silent: opts?.silent, formatMode: tg.formatMode, kindMetadata: opts?.kindMetadata,
                                     send: command => meshClient.send({ machineId: holder, url }, command, 0, { timeoutMs: relayTimeoutMs }) });